diff --git a/.gitignore b/.gitignore index 2f5ff94b5..4e4b7c08d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,10 @@ /easy-rsa-master/ /easy-rsa.tar.gz /easy-rsa + +# editor and IDE paraphernalia +.idea +.vscode + +# macOS paraphernalia +.DS_Store diff --git a/Makefile b/Makefile index 334c1d1b2..e29169794 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ endif GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) INSTALL_LOCATION:=$(shell go env GOPATH)/bin -GOLANGCI_LINT_VERSION ?= 1.45.2 +GOLANGCI_LINT_VERSION ?= 1.50.1 GOSEC_VERSION ?= 2.13.1 REGISTRY ?= gcr.io/$(shell gcloud config get-value project) @@ -56,7 +56,8 @@ mock_gen: .PHONY: test test: - GO111MODULE=on go test -race sigs.k8s.io/apiserver-network-proxy/... + go test -race ./... + cd konnectivity-client && go test -race ./... ## -------------------------------------- ## Binaries @@ -90,7 +91,7 @@ bin/proxy-server-static: proto/agent/agent.pb.go konnectivity-client/proto/clien .PHONY: lint lint: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(INSTALL_LOCATION) v$(GOLANGCI_LINT_VERSION) - $(INSTALL_LOCATION)/golangci-lint run --no-config --disable-all --enable=gofmt,golint,gosec,govet,unused --fix --verbose --timeout 3m + $(INSTALL_LOCATION)/golangci-lint run --no-config --disable-all --enable=gofmt,revive,gosec,govet,unused --fix --verbose --timeout 3m ## -------------------------------------- ## Go diff --git a/OWNERS b/OWNERS index d97517295..50ea85c9a 100644 --- a/OWNERS +++ b/OWNERS @@ -1,15 +1,15 @@ # See the OWNERS docs at https://go.k8s.io/owners reviewers: - - caesarxuchao - dberkov - jefftree - - jkh52 + - tallclair approvers: - - cheftako - - mcrute - anfernee + - cheftako + - jkh52 +emeritus_approvers: - andrewsykim - caesarxuchao -emeritus_approvers: + - mcrute - Sh4d1 diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index e208979ec..27517e896 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package options import ( @@ -30,10 +46,14 @@ type GrpcProxyAgentOptions struct { ProxyServerPort int AlpnProtos []string - // Ports for the health and admin server + // Bind address for the health connections. HealthServerHost string + // Port we listen for health connections on. HealthServerPort int - AdminServerPort int + // Bind address for the admin connections. + AdminBindAddress string + // Port we listen for admin connections on. + AdminServerPort int // Enables pprof at host:adminPort/debug/pprof. EnableProfiling bool // If EnableProfiling is true, this enables the lock contention @@ -59,9 +79,9 @@ type GrpcProxyAgentOptions struct { // The check is an "unlocked" read but is still use at your own peril. WarnOnChannelLimit bool - SyncForever bool - BindAddress string - ApiServerMapping portMapping + SyncForever bool + BindAddress string + APIServerMapping portMapping } var _ pflag.Value = &portMapping{} @@ -108,6 +128,7 @@ func (o *GrpcProxyAgentOptions) Flags() *pflag.FlagSet { flags.StringVar(&o.HealthServerHost, "health-server-host", o.HealthServerHost, "The host address to listen on, without port.") flags.IntVar(&o.HealthServerPort, "health-server-port", o.HealthServerPort, "The port the health server is listening on.") flags.IntVar(&o.AdminServerPort, "admin-server-port", o.AdminServerPort, "The port the admin server is listening on.") + flags.StringVar(&o.AdminBindAddress, "admin-bind-address", o.AdminBindAddress, "Bind address for admin connections. If empty, we will bind to all interfaces.") flags.BoolVar(&o.EnableProfiling, "enable-profiling", o.EnableProfiling, "enable pprof at host:admin-port/debug/pprof") flags.BoolVar(&o.EnableContentionProfiling, "enable-contention-profiling", o.EnableContentionProfiling, "enable contention profiling at host:admin-port/debug/pprof/block. \"--enable-profiling\" must also be set.") flags.StringVar(&o.AgentID, "agent-id", o.AgentID, "The unique ID of this agent. Can also be set by the 'PROXY_AGENT_ID' environment variable. Default to a generated uuid if not set.") @@ -118,7 +139,7 @@ func (o *GrpcProxyAgentOptions) Flags() *pflag.FlagSet { flags.StringVar(&o.ServiceAccountTokenPath, "service-account-token-path", o.ServiceAccountTokenPath, "If non-empty proxy agent uses this token to prove its identity to the proxy server.") flags.StringVar(&o.AgentIdentifiers, "agent-identifiers", o.AgentIdentifiers, "Identifiers of the agent that will be used by the server when choosing agent. N.B. the list of identifiers must be in URL encoded format. e.g.,host=localhost&host=node1.mydomain.com&cidr=127.0.0.1/16&ipv4=1.2.3.4&ipv4=5.6.7.8&ipv6=:::::&default-route=true") flags.BoolVar(&o.WarnOnChannelLimit, "warn-on-channel-limit", o.WarnOnChannelLimit, "Turns on a warning if the system is going to push to a full channel. The check involves an unsafe read.") - flags.Var(&o.ApiServerMapping, "apiserver-port-mapping", "Mapping between a local port and the host:port used to reach the Kubernetes API Server") + flags.Var(&o.APIServerMapping, "apiserver-port-mapping", "Mapping between a local port and the host:port used to reach the Kubernetes API Server") flags.StringVar(&o.BindAddress, "bind-address", o.BindAddress, "Address used to listen for traffic generated on cluster network") // add feature gates flag features.DefaultMutableFeatureGate.AddFlag(flags) @@ -135,6 +156,7 @@ func (o *GrpcProxyAgentOptions) Print() { klog.V(1).Infof("ALPNProtos set to %+s.\n", o.AlpnProtos) klog.V(1).Infof("HealthServerHost set to %s\n", o.HealthServerHost) klog.V(1).Infof("HealthServerPort set to %d.\n", o.HealthServerPort) + klog.V(1).Infof("Admin bind address set to %q.\n", o.AdminBindAddress) klog.V(1).Infof("AdminServerPort set to %d.\n", o.AdminServerPort) klog.V(1).Infof("EnableProfiling set to %v.\n", o.EnableProfiling) klog.V(1).Infof("EnableContentionProfiling set to %v.\n", o.EnableContentionProfiling) @@ -148,7 +170,7 @@ func (o *GrpcProxyAgentOptions) Print() { klog.V(1).Infof("WarnOnChannelLimit set to %t.\n", o.WarnOnChannelLimit) if features.DefaultMutableFeatureGate.Enabled(features.NodeToMasterTraffic) { klog.V(1).Infof("AgentBindAddress set to %s.\n", o.BindAddress) - klog.V(1).Infof("Apiserver port mapping set to %s.\n", o.ApiServerMapping.String()) + klog.V(1).Infof("Apiserver port mapping set to %s.\n", o.APIServerMapping.String()) } klog.V(1).Infof("SyncForever set to %v.\n", o.SyncForever) } @@ -201,20 +223,20 @@ func (o *GrpcProxyAgentOptions) Validate() error { if err := validateHostnameOrIP(o.BindAddress); err != nil { return fmt.Errorf("agent bind address is invalid: %v", err) } - if err := validateHostnameOrIP(o.ApiServerMapping.RemoteHost); err != nil { + if err := validateHostnameOrIP(o.APIServerMapping.RemoteHost); err != nil { return fmt.Errorf("apiserver address is invalid: %v", err) } - if o.ApiServerMapping.LocalPort > 49151 { - return fmt.Errorf("please do not try to use ephemeral port %d for the apiserver local port", o.ApiServerMapping.LocalPort) + if o.APIServerMapping.LocalPort > 49151 { + return fmt.Errorf("please do not try to use ephemeral port %d for the apiserver local port", o.APIServerMapping.LocalPort) } - if o.ApiServerMapping.LocalPort < 1024 { - return fmt.Errorf("please do not try to use reserved port %d for the apiserver local port", o.ApiServerMapping.LocalPort) + if o.APIServerMapping.LocalPort < 1024 { + return fmt.Errorf("please do not try to use reserved port %d for the apiserver local port", o.APIServerMapping.LocalPort) } - if o.ApiServerMapping.RemotePort > 49151 { - return fmt.Errorf("please do not try to use ephemeral port %d for the apiserver remote port", o.ApiServerMapping.LocalPort) + if o.APIServerMapping.RemotePort > 49151 { + return fmt.Errorf("please do not try to use ephemeral port %d for the apiserver remote port", o.APIServerMapping.LocalPort) } - if o.ApiServerMapping.RemotePort < 1 { - return fmt.Errorf("invalid port %d for the apiserver remote port", o.ApiServerMapping.RemotePort) + if o.APIServerMapping.RemotePort < 1 { + return fmt.Errorf("invalid port %d for the apiserver remote port", o.APIServerMapping.RemotePort) } return nil } @@ -259,6 +281,7 @@ func NewGrpcProxyAgentOptions() *GrpcProxyAgentOptions { ProxyServerPort: 8091, HealthServerHost: "", HealthServerPort: 8093, + AdminBindAddress: "127.0.0.1", AdminServerPort: 8094, EnableProfiling: false, EnableContentionProfiling: false, @@ -271,7 +294,7 @@ func NewGrpcProxyAgentOptions() *GrpcProxyAgentOptions { ServiceAccountTokenPath: "", WarnOnChannelLimit: false, SyncForever: false, - ApiServerMapping: portMapping{LocalPort: 6443, RemoteHost: "localhost", RemotePort: 6443}, + APIServerMapping: portMapping{LocalPort: 6443, RemoteHost: "localhost", RemotePort: 6443}, BindAddress: "127.0.0.1", } return &o diff --git a/cmd/agent/app/options/options_test.go b/cmd/agent/app/options/options_test.go new file mode 100644 index 000000000..bce6fe933 --- /dev/null +++ b/cmd/agent/app/options/options_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "reflect" + "testing" + "time" +) + +/* + * TestDefaultServerOptions is intended to ensure we do not make a backward incompatible + * change to the default flag values for the ANP agent. + */ +func TestDefaultServerOptions(t *testing.T) { + defaultAgentOptions := NewGrpcProxyAgentOptions() + assertDefaultValue(t, "AgentCert", defaultAgentOptions.AgentCert, "") + assertDefaultValue(t, "AgentKey", defaultAgentOptions.AgentKey, "") + assertDefaultValue(t, "CaCert", defaultAgentOptions.CaCert, "") + assertDefaultValue(t, "ProxyServerHost", defaultAgentOptions.ProxyServerHost, "127.0.0.1") + assertDefaultValue(t, "ProxyServerPort", defaultAgentOptions.ProxyServerPort, 8091) + assertDefaultValue(t, "HealthServerHost", defaultAgentOptions.HealthServerHost, "") + assertDefaultValue(t, "HealthServerPort", defaultAgentOptions.HealthServerPort, 8093) + assertDefaultValue(t, "AdminBindAddress", defaultAgentOptions.AdminBindAddress, "127.0.0.1") + assertDefaultValue(t, "AdminServerPort", defaultAgentOptions.AdminServerPort, 8094) + assertDefaultValue(t, "EnableProfiling", defaultAgentOptions.EnableProfiling, false) + assertDefaultValue(t, "EnableContentionProfiling", defaultAgentOptions.EnableContentionProfiling, false) + assertDefaultValue(t, "AgentIdentifiers", defaultAgentOptions.AgentIdentifiers, "") + assertDefaultValue(t, "SyncInterval", defaultAgentOptions.SyncInterval, 1*time.Second) + assertDefaultValue(t, "ProbeInterval", defaultAgentOptions.ProbeInterval, 1*time.Second) + assertDefaultValue(t, "SyncIntervalCap", defaultAgentOptions.SyncIntervalCap, 10*time.Second) + assertDefaultValue(t, "KeepaliveTime", defaultAgentOptions.KeepaliveTime, 1*time.Hour) + assertDefaultValue(t, "ServiceAccountTokenPath", defaultAgentOptions.ServiceAccountTokenPath, "") + assertDefaultValue(t, "WarnOnChannelLimit", defaultAgentOptions.WarnOnChannelLimit, false) + assertDefaultValue(t, "SyncForever", defaultAgentOptions.SyncForever, false) +} + +func assertDefaultValue(t *testing.T, fieldName string, actual, expected interface{}) { + t.Helper() + assert.IsType(t, expected, actual, "For field %s, got the wrong type.", fieldName) + assert.Equal(t, expected, actual, "For field %s, got the wrong value.", fieldName) +} + +func TestValidate(t *testing.T) { + for desc, tc := range map[string]struct { + fieldMap map[string]interface{} + expected error + }{ + "default": { + fieldMap: map[string]interface{}{}, + expected: nil, + }, + "ZeroProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 0}, + expected: fmt.Errorf("proxy server port 0 must be greater than 0"), + }, + "NegativeProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": -1}, + expected: fmt.Errorf("proxy server port -1 must be greater than 0"), + }, + "ReservedProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 1023}, + expected: nil, //TODO: fmt.Errorf("please do not try to use reserved port 1023 for the proxy server port"), + }, + "StartValidProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 1024}, + expected: nil, + }, + "EndValidProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 49151}, + expected: nil, + }, + "StartEphemeralProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 49152}, + expected: nil, //TODO: fmt.Errorf("please do not try to use ephemeral port 49152 for the proxy server port"), + }, + "ZeroHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 0}, + expected: fmt.Errorf("health server port 0 must be greater than 0"), + }, + "NegativeHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": -1}, + expected: fmt.Errorf("health server port -1 must be greater than 0"), + }, + "ReservedHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 1023}, + expected: nil, //TODO: fmt.Errorf("please do not try to use reserved port 1023 for the health server port"), + }, + "StartValidHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 1024}, + expected: nil, + }, + "EndValidHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 49151}, + expected: nil, + }, + "StartEphemeralHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 49152}, + expected: nil, //TODO: fmt.Errorf("please do not try to use ephemeral port 49152 for the health server port"), + }, + "ZeroAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 0}, + expected: fmt.Errorf("admin server port 0 must be greater than 0"), + }, + "NegativeAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": -1}, + expected: fmt.Errorf("admin server port -1 must be greater than 0"), + }, + "ReservedAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 1023}, + expected: nil, //TODO: fmt.Errorf("please do not try to use reserved port 1023 for the health port"), + }, + "StartValidAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 1024}, + expected: nil, + }, + "EndValidAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 49151}, + expected: nil, + }, + "StartEphemeralAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 49152}, + expected: nil, //TODO: fmt.Errorf("please do not try to use ephemeral port 49152 for the health port"), + }, + "ContentionProfilingRequiresProfiling": { + fieldMap: map[string]interface{}{ + "EnableContentionProfiling": true, + "EnableProfiling": false, + }, + expected: fmt.Errorf("if --enable-contention-profiling is set, --enable-profiling must also be set"), + }, + } { + t.Run(desc, func(t *testing.T) { + testAgentOptions := NewGrpcProxyAgentOptions() + for field, value := range tc.fieldMap { + rv := reflect.ValueOf(testAgentOptions) + rv = rv.Elem() + fv := rv.FieldByName(field) + switch reflect.TypeOf(value).Kind() { + case reflect.String: + svalue := value.(string) + fv.SetString(svalue) + case reflect.Int: + ivalue := value.(int) + fv.SetInt(int64(ivalue)) + case reflect.Bool: + bvalue := value.(bool) + fv.SetBool(bvalue) + } + } + actual := testAgentOptions.Validate() + assert.IsType(t, tc.expected, actual, "Validation for case %s, got the wrong type.", desc) + assert.Equal(t, tc.expected, actual, "Validation for case %s, got the wrong value.", desc) + }) + } +} diff --git a/cmd/agent/app/server.go b/cmd/agent/app/server.go index c75ae571f..130fab216 100644 --- a/cmd/agent/app/server.go +++ b/cmd/agent/app/server.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package app import ( @@ -99,8 +115,8 @@ func (a *Agent) runControlPlaneProxy(ctx context.Context, o *options.GrpcProxyAg ClientSet: cs, ListenHost: o.BindAddress, } - klog.V(1).Infof("Exposing apiserver: %s", &o.ApiServerMapping) - return pf.Serve(ctx, agent.PortMapping(o.ApiServerMapping)) + klog.V(1).Infof("Exposing apiserver: %s", &o.APIServerMapping) + return pf.Serve(ctx, agent.PortMapping(o.APIServerMapping)) } func (a *Agent) runHealthServer(o *options.GrpcProxyAgentOptions) error { @@ -155,13 +171,16 @@ func (a *Agent) runAdminServer(o *options.GrpcProxyAgentOptions) error { if o.EnableProfiling { muxHandler.HandleFunc("/debug/pprof", util.RedirectTo("/debug/pprof/")) muxHandler.HandleFunc("/debug/pprof/", pprof.Index) + muxHandler.HandleFunc("/debug/pprof/profile", pprof.Profile) + muxHandler.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + muxHandler.HandleFunc("/debug/pprof/trace", pprof.Trace) if o.EnableContentionProfiling { runtime.SetBlockProfileRate(1) } } adminServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", o.AdminServerPort), + Addr: net.JoinHostPort(o.AdminBindAddress, strconv.Itoa(o.AdminServerPort)), Handler: muxHandler, MaxHeaderBytes: 1 << 20, ReadHeaderTimeout: ReadHeaderTimeout, diff --git a/cmd/server/app/options/options.go b/cmd/server/app/options/options.go index adcc663fa..062c361b7 100644 --- a/cmd/server/app/options/options.go +++ b/cmd/server/app/options/options.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package options import ( diff --git a/cmd/server/app/options/options_test.go b/cmd/server/app/options/options_test.go index ad242e4ed..0598bb977 100644 --- a/cmd/server/app/options/options_test.go +++ b/cmd/server/app/options/options_test.go @@ -1,13 +1,33 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package options import ( - "testing" - "reflect" - "time" "fmt" "github.com/stretchr/testify/assert" + "reflect" + "testing" + "time" ) +/* + * TestDefaultServerOptions is intended to ensure we do not make a backward incompatible + * change to the default flag values for the ANP server. + */ func TestDefaultServerOptions(t *testing.T) { defaultServerOptions := NewProxyRunOptions() assertDefaultValue(t, "ServerCert", defaultServerOptions.ServerCert, "") diff --git a/cmd/server/app/server.go b/cmd/server/app/server.go index 59829803f..7073c7c15 100644 --- a/cmd/server/app/server.go +++ b/cmd/server/app/server.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package app import ( @@ -375,14 +391,18 @@ func (p *Proxy) runAdminServer(o *options.ProxyRunOptions, server *server.ProxyS if o.EnableProfiling { muxHandler.HandleFunc("/debug/pprof", util.RedirectTo("/debug/pprof/")) muxHandler.HandleFunc("/debug/pprof/", netpprof.Index) + muxHandler.HandleFunc("/debug/pprof/profile", netpprof.Profile) + muxHandler.HandleFunc("/debug/pprof/symbol", netpprof.Symbol) + muxHandler.HandleFunc("/debug/pprof/trace", netpprof.Trace) if o.EnableContentionProfiling { runtime.SetBlockProfileRate(1) } } adminServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", o.AdminPort), - Handler: muxHandler, - MaxHeaderBytes: 1 << 20, + Addr: net.JoinHostPort(o.AdminBindAddress, strconv.Itoa(o.AdminPort)), + Handler: muxHandler, + MaxHeaderBytes: 1 << 20, + ReadHeaderTimeout: ReadHeaderTimeout, } labels := runpprof.Labels( diff --git a/go.mod b/go.mod index c2e1270a3..5c34d629b 100644 --- a/go.mod +++ b/go.mod @@ -13,32 +13,34 @@ require ( go.uber.org/goleak v1.2.0 golang.org/x/net v0.4.0 google.golang.org/grpc v1.48.0 - k8s.io/api v0.24.3 - k8s.io/apimachinery v0.24.3 - k8s.io/client-go v0.24.3 - k8s.io/component-base v0.24.3 - k8s.io/klog/v2 v2.70.1 + k8s.io/api v0.24.8 + k8s.io/apimachinery v0.24.8 + k8s.io/client-go v0.24.8 + k8s.io/component-base v0.24.8 + k8s.io/klog/v2 v2.60.1 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.0 ) require ( + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.22.0 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/gnostic v0.6.9 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/imdario/mergo v0.3.13 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/imdario/mergo v0.3.5 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.7.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -54,16 +56,16 @@ require ( golang.org/x/text v0.5.0 // indirect golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220812140447-cec7f5303424 // indirect + google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea // indirect - k8s.io/utils v0.0.0-20220812165043-ad590609e2e5 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect ) replace sigs.k8s.io/apiserver-network-proxy/konnectivity-client => ./konnectivity-client diff --git a/go.sum b/go.sum index c4d029440..24dc94be4 100644 --- a/go.sum +++ b/go.sum @@ -48,7 +48,9 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -65,7 +67,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -93,9 +94,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -108,10 +108,10 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -128,21 +128,18 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.0 h1:1VXunYCNgapcSzFtcY+eBmrwESlYCnFJZahQRgTRoo8= -github.com/go-openapi/swag v0.22.0/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -183,9 +180,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= -github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -200,9 +196,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -232,12 +227,10 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -256,7 +249,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -264,9 +256,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -286,18 +277,19 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -357,15 +349,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -397,7 +385,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -421,9 +408,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -435,7 +421,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -476,8 +461,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -496,9 +479,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7 h1:dtndE8FcEta75/4kHF3AbpuWzV6f1LjnLrM4pe2SZrw= -golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -563,7 +545,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -587,9 +568,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -648,6 +628,7 @@ golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -719,9 +700,8 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220812140447-cec7f5303424 h1:zZnTt15U44/Txe/9cN/tVbteBkPMiyXK48hPsKRmqj4= -google.golang.org/genproto v0.0.0-20220812140447-cec7f5303424/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -764,12 +744,13 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -783,7 +764,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= @@ -795,37 +775,31 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.24.3 h1:tt55QEmKd6L2k5DP6G/ZzdMQKvG5ro4H4teClqm0sTY= -k8s.io/api v0.24.3/go.mod h1:elGR/XSZrS7z7cSZPzVWaycpJuGIw57j9b95/1PdJNI= -k8s.io/apimachinery v0.24.3 h1:hrFiNSA2cBZqllakVYyH/VyEh4B581bQRmqATJSeQTg= -k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/client-go v0.24.3 h1:Nl1840+6p4JqkFWEW2LnMKU667BUxw03REfLAVhuKQY= -k8s.io/client-go v0.24.3/go.mod h1:AAovolf5Z9bY1wIg2FZ8LPQlEdKHjLI7ZD4rw920BJw= -k8s.io/component-base v0.24.3 h1:u99WjuHYCRJjS1xeLOx72DdRaghuDnuMgueiGMFy1ec= -k8s.io/component-base v0.24.3/go.mod h1:bqom2IWN9Lj+vwAkPNOv2TflsP1PeVDIwIN0lRthxYY= +k8s.io/api v0.24.8 h1:5bZ6aotI1J+BG2g0U9nBrPy1dOzBeJ/HFOHi6dX+ud4= +k8s.io/api v0.24.8/go.mod h1:W2RSRCK+eDrYEH4YeuSKrIY90TYYIcW1ojk8Mo6HVOo= +k8s.io/apimachinery v0.24.8 h1:/xehDgfpC4uN7I1GBVvx+Anwb2Jjem+GyJ9F9lSmkBs= +k8s.io/apimachinery v0.24.8/go.mod h1:WR5z9Lpw2mOAeDg20iSSrEBRQMY0p2YXVdYpUIgSr4o= +k8s.io/client-go v0.24.8 h1:drY++vKWtstFnG5Z1/4KkViBWRFaaiUugOfM4emNvyE= +k8s.io/client-go v0.24.8/go.mod h1:YrQVH7HVsT8VIsFgzLUrXHVq9RRt13hA9RoDQ8It76g= +k8s.io/component-base v0.24.8 h1:3qjRp7V+lj8b3HmZP6kF21EaKU0femkj78Gvw7cPU5M= +k8s.io/component-base v0.24.8/go.mod h1:jYVLUn7NDJs0a8CtuPSudgLEuW/PMrjeCALZZCS7K2E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea h1:3QOH5+2fGsY8e1qf+GIFpg+zw/JGNrgyZRQR7/m6uWg= -k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220812165043-ad590609e2e5 h1:XmRqFcQlCy/lKRZ39j+RVpokYNroHPqV3mcBRfnhT5o= -k8s.io/utils v0.0.0-20220812165043-ad590609e2e5/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/konnectivity-client/go.mod b/konnectivity-client/go.mod index e1b08efb9..80fe85943 100644 --- a/konnectivity-client/go.mod +++ b/konnectivity-client/go.mod @@ -6,18 +6,24 @@ go 1.17 // k/k minor version, to prevent client backport issues. require ( github.com/golang/protobuf v1.4.3 - go.uber.org/goleak v1.2.0 + github.com/prometheus/client_golang v1.0.0 + go.uber.org/goleak v1.1.10 google.golang.org/grpc v1.27.1 k8s.io/klog/v2 v2.0.0 ) require ( + github.com/beorn7/perks v1.0.0 // indirect github.com/go-logr/logr v0.1.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect + github.com/prometheus/common v0.4.1 // indirect + github.com/prometheus/procfs v0.0.2 // indirect golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect - golang.org/x/text v0.3.3 // indirect - golang.org/x/tools v0.1.5 // indirect - google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect - google.golang.org/protobuf v1.26.0-rc.1 // indirect + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a // indirect + golang.org/x/text v0.3.0 // indirect + golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 // indirect + google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect + google.golang.org/protobuf v1.23.0 // indirect ) diff --git a/konnectivity-client/go.sum b/konnectivity-client/go.sum index ecbd23264..373f9a9aa 100644 --- a/konnectivity-client/go.sum +++ b/konnectivity-client/go.sum @@ -1,47 +1,79 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -50,49 +82,39 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -100,13 +122,12 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/konnectivity-client/pkg/client/client.go b/konnectivity-client/pkg/client/client.go index c55070ae6..cb186cefc 100644 --- a/konnectivity-client/pkg/client/client.go +++ b/konnectivity-client/pkg/client/client.go @@ -29,6 +29,9 @@ import ( "google.golang.org/grpc" "k8s.io/klog/v2" + + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) @@ -131,6 +134,9 @@ type grpcTunnel struct { // closing should only be accessed through atomic methods. // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+ closing uint32 + + // Stores the current metrics.ClientConnectionStatus + prevStatus atomic.Value } type clientConn interface { @@ -139,6 +145,11 @@ type clientConn interface { var _ clientConn = &grpc.ClientConn{} +var ( + // Expose metrics for client to register. + Metrics = metrics.Metrics +) + // CreateSingleUseGrpcTunnel creates a Tunnel to dial to a remote server through a // gRPC based proxy service. // Currently, a single tunnel supports a single connection, and the tunnel is closed when the connection is terminated @@ -177,7 +188,7 @@ func CreateSingleUseGrpcTunnelWithContext(createCtx, tunnelCtx context.Context, } func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *grpcTunnel { - return &grpcTunnel{ + t := grpcTunnel{ stream: stream, clientConn: c, pendingDial: pendingDialManager{pendingDials: make(map[int64]pendingDial)}, @@ -185,6 +196,36 @@ func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *g readTimeoutSeconds: 10, done: make(chan struct{}), } + s := metrics.ClientConnectionStatusCreated + t.prevStatus.Store(s) + metrics.Metrics.GetClientConnectionsMetric().WithLabelValues(string(s)).Inc() + return &t +} + +func (t *grpcTunnel) updateMetric(status metrics.ClientConnectionStatus) { + select { + case <-t.Done(): + return + default: + } + + prevStatus := t.prevStatus.Swap(status).(metrics.ClientConnectionStatus) + + m := metrics.Metrics.GetClientConnectionsMetric() + m.WithLabelValues(string(prevStatus)).Dec() + m.WithLabelValues(string(status)).Inc() +} + +// closeMetric should be called exactly once to finalize client_connections metric. +func (t *grpcTunnel) closeMetric() { + select { + case <-t.Done(): + return + default: + } + prevStatus := t.prevStatus.Load().(metrics.ClientConnectionStatus) + + metrics.Metrics.GetClientConnectionsMetric().WithLabelValues(string(prevStatus)).Dec() } func (t *grpcTunnel) serve(tunnelCtx context.Context) { @@ -196,19 +237,29 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { // close any channels remaining for these connections. t.conns.closeAll() + t.closeMetric() + close(t.done) }() for { pkt, err := t.stream.Recv() - if err == io.EOF || t.isClosing() { + if err == io.EOF { return } + const segment = commonmetrics.SegmentToClient + isClosing := t.isClosing() if err != nil || pkt == nil { - klog.ErrorS(err, "stream read failure") + if !isClosing { + klog.ErrorS(err, "stream read failure") + } + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) + return + } + metrics.Metrics.ObservePacket(segment, pkt.Type) + if isClosing { return } - klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type) switch pkt.Type { @@ -222,13 +273,19 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { // 2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context // // In either scenario, we should return here and close the tunnel as it is no longer needed. - klog.V(1).InfoS("DialResp not recognized; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random) + kvs := []interface{}{"dialID", resp.Random, "connectID", resp.ConnectID} + if resp.Error != "" { + kvs = append(kvs, "error", resp.Error) + } + klog.V(1).InfoS("DialResp not recognized; dropped", kvs...) return } result := dialResult{connid: resp.ConnectID} if resp.Error != "" { - result.err = &dialFailure{resp.Error, DialFailureEndpoint} + result.err = &dialFailure{resp.Error, metrics.DialFailureEndpoint} + } else { + t.updateMetric(metrics.ClientConnectionStatusOk) } select { // try to send to the result channel @@ -263,7 +320,7 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { klog.V(1).InfoS("DIAL_CLS after dial finished", "dialID", resp.Random) } else { result := dialResult{ - err: &dialFailure{"dial closed", DialFailureDialClosed}, + err: &dialFailure{"dial closed", metrics.DialFailureDialClosed}, } select { case pendingDial.resultCh <- result: @@ -316,6 +373,15 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { // Dial connects to the address on the named network, similar to // what net.Dial does. The only supported protocol is tcp. func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) { + conn, err := t.dialContext(requestCtx, protocol, address) + if err != nil { + _, reason := GetDialFailureReason(err) + metrics.Metrics.ObserveDialFailure(reason) + } + return conn, err +} + +func (t *grpcTunnel) dialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) { select { case <-t.done: return nil, errors.New("tunnel is closed") @@ -326,6 +392,8 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s return nil, errors.New("protocol not supported") } + t.updateMetric(metrics.ClientConnectionStatusDialing) + random := rand.Int63() /* #nosec G404 */ // This channel is closed once we're returning and no longer waiting on resultCh @@ -350,8 +418,11 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s } klog.V(5).InfoS("[tracing] send packet", "type", req.Type) + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, req.Type) err := t.stream.Send(req) if err != nil { + metrics.Metrics.ObserveStreamError(segment, err, req.Type) return nil, err } @@ -375,14 +446,14 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s case <-time.After(30 * time.Second): klog.V(5).InfoS("Timed out waiting for DialResp", "dialID", random) go t.closeDial(random) - return nil, &dialFailure{"dial timeout, backstop", DialFailureTimeout} + return nil, &dialFailure{"dial timeout, backstop", metrics.DialFailureTimeout} case <-requestCtx.Done(): klog.V(5).InfoS("Context canceled waiting for DialResp", "ctxErr", requestCtx.Err(), "dialID", random) go t.closeDial(random) - return nil, &dialFailure{"dial timeout, context", DialFailureContext} + return nil, &dialFailure{"dial timeout, context", metrics.DialFailureContext} case <-t.done: klog.V(5).InfoS("Tunnel closed while waiting for DialResp", "dialID", random) - return nil, &dialFailure{"tunnel closed", DialFailureTunnelClosed} + return nil, &dialFailure{"tunnel closed", metrics.DialFailureTunnelClosed} } return c, nil @@ -402,7 +473,10 @@ func (t *grpcTunnel) closeDial(dialID int64) { }, }, } + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, req.Type) if err := t.stream.Send(req); err != nil { + metrics.Metrics.ObserveStreamError(segment, err, req.Type) klog.V(5).InfoS("Failed to send DIAL_CLS", "err", err, "dialID", dialID) } t.closeTunnel() @@ -417,38 +491,19 @@ func (t *grpcTunnel) isClosing() bool { return atomic.LoadUint32(&t.closing) != 0 } -func GetDialFailureReason(err error) (isDialFailure bool, reason DialFailureReason) { +func GetDialFailureReason(err error) (isDialFailure bool, reason metrics.DialFailureReason) { var df *dialFailure if errors.As(err, &df) { return true, df.reason } - return false, DialFailureUnknown + return false, metrics.DialFailureUnknown } type dialFailure struct { msg string - reason DialFailureReason + reason metrics.DialFailureReason } func (df *dialFailure) Error() string { return df.msg } - -type DialFailureReason string - -const ( - DialFailureUnknown DialFailureReason = "unknown" - // DialFailureTimeout indicates the hard 30 second timeout was hit. - DialFailureTimeout DialFailureReason = "timeout" - // DialFailureContext indicates that the context was cancelled or reached it's deadline before - // the dial response was returned. - DialFailureContext DialFailureReason = "context" - // DialFailureEndpoint indicates that the konnectivity-agent was unable to reach the backend endpoint. - DialFailureEndpoint DialFailureReason = "endpoint" - // DialFailureDialClosed indicates that the client received a CloseDial response, indicating the - // connection was closed before the dial could complete. - DialFailureDialClosed DialFailureReason = "dialclosed" - // DialFailureTunnelClosed indicates that the client connection was closed before the dial could - // complete. - DialFailureTunnelClosed DialFailureReason = "tunnelclosed" -) diff --git a/konnectivity-client/pkg/client/client_test.go b/konnectivity-client/pkg/client/client_test.go index a4ef71176..3ac773b58 100644 --- a/konnectivity-client/pkg/client/client_test.go +++ b/konnectivity-client/pkg/client/client_test.go @@ -26,9 +26,12 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/goleak" "google.golang.org/grpc" "k8s.io/klog/v2" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + metricstest "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics/testing" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) @@ -36,12 +39,13 @@ func TestMain(m *testing.M) { fs := flag.NewFlagSet("test", flag.PanicOnError) klog.InitFlags(fs) fs.Set("v", "9") + metrics.Metrics.RegisterMetrics(prometheus.DefaultRegisterer) m.Run() } func TestDial(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -52,6 +56,10 @@ func TestDial(t *testing.T) { tunnel := newUnstartedTunnel(s, s.conn()) + if err := metricstest.ExpectClientConnection(metrics.ClientConnectionStatusCreated, 1); err != nil { + t.Error(err) + } + go tunnel.serve(ctx) go ts.serve() @@ -67,12 +75,24 @@ func TestDial(t *testing.T) { if ts.packets[0].GetDialRequest().Address != "127.0.0.1:80" { t.Errorf("expect packet.address %v; got %v", "127.0.0.1:80", ts.packets[0].GetDialRequest().Address) } + + if err := metricstest.ExpectClientConnections(map[metrics.ClientConnectionStatus]int{ + metrics.ClientConnectionStatusCreated: 0, + metrics.ClientConnectionStatusDialing: 0, + metrics.ClientConnectionStatusOk: 1, + }); err != nil { + t.Error(err) + } + if err := metricstest.ExpectClientDialFailures(nil); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. } // TestDialRace exercises the scenario where serve() observes and handles DIAL_RSP // before DialContext() does any work after sending the DIAL_REQ. func TestDialRace(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -100,6 +120,10 @@ func TestDialRace(t *testing.T) { if ts.packets[0].GetDialRequest().Address != "127.0.0.1:80" { t.Errorf("expect packet.address %v; got %v", "127.0.0.1:80", ts.packets[0].GetDialRequest().Address) } + + if err := metricstest.ExpectClientDialFailures(nil); err != nil { + t.Error(err) + } } // fakeSlowSend wraps ProxyService_ProxyClient and adds an artificial 1 second delay after calling Send @@ -118,7 +142,7 @@ func (s fakeSlowSend) Send(p *client.Packet) error { } func TestData(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -174,7 +198,7 @@ func TestData(t *testing.T) { } func TestClose(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -193,6 +217,14 @@ func TestClose(t *testing.T) { t.Fatalf("expect nil; got %v", err) } + if err := metricstest.ExpectClientConnections(map[metrics.ClientConnectionStatus]int{ + metrics.ClientConnectionStatusCreated: 0, + metrics.ClientConnectionStatusDialing: 0, + metrics.ClientConnectionStatusOk: 1, + }); err != nil { + t.Error(err) + } + if err := conn.Close(); err != nil { t.Error(err) } @@ -203,13 +235,23 @@ func TestClose(t *testing.T) { if ts.packets[1].GetCloseRequest().ConnectID != 100 { t.Errorf("expect connectID=100; got %d", ts.packets[1].GetCloseRequest().ConnectID) } + + <-tunnel.Done() + if err := metricstest.ExpectClientConnections(map[metrics.ClientConnectionStatus]int{ + metrics.ClientConnectionStatusCreated: 0, + metrics.ClientConnectionStatusDialing: 0, + metrics.ClientConnectionStatusOk: 0, + }); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. } func TestCloseTimeout(t *testing.T) { if testing.Short() { t.Skip() } - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -246,6 +288,15 @@ func TestCloseTimeout(t *testing.T) { t.Errorf("expected %v but got %v", errConnCloseTimeout, err) } + <-tunnel.Done() + if err := metricstest.ExpectClientConnections(map[metrics.ClientConnectionStatus]int{ + metrics.ClientConnectionStatusCreated: 0, + metrics.ClientConnectionStatusDialing: 0, + metrics.ClientConnectionStatusOk: 0, + }); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. } func TestCreateSingleUseGrpcTunnel_NoLeakOnFailure(t *testing.T) { @@ -273,7 +324,7 @@ func TestCreateSingleUseGrpcTunnelWithContext_NoLeakOnFailure(t *testing.T) { } func TestDialAfterTunnelCancelled(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -293,6 +344,9 @@ func TestDialAfterTunnelCancelled(t *testing.T) { t.Fatalf("expect err when dialing after tunnel closed") } + // TODO(jkh52): verify dial failure metric. + metrics.Metrics.Reset() // For clean shutdown. + select { case <-tunnel.Done(): case <-time.After(30 * time.Second): @@ -301,7 +355,7 @@ func TestDialAfterTunnelCancelled(t *testing.T) { } func TestDial_RequestContextCancelled(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) s, ps := pipe() defer ps.Close() @@ -321,9 +375,6 @@ func TestDial_RequestContextCancelled(t *testing.T) { go ts.serve() func() { - // Tunnel should be shut down when the dial fails. - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) - tunnel := newUnstartedTunnel(s, s.conn()) go tunnel.serve(context.Background()) @@ -335,10 +386,15 @@ func TestDial_RequestContextCancelled(t *testing.T) { isDialFailure, reason := GetDialFailureReason(err) if !isDialFailure { t.Errorf("Unexpected non-dial failure error: %v", err) - } else if reason != DialFailureContext { + } else if reason != metrics.DialFailureContext { t.Errorf("Expected DialFailureContext, got %v", reason) } + if err := metricstest.ExpectClientDialFailure(metrics.DialFailureContext, 1); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. + ts.assertPacketType(0, client.PacketType_DIAL_REQ) waitForDialClsStart := time.Now() select { @@ -360,7 +416,7 @@ func TestDial_RequestContextCancelled(t *testing.T) { } func TestDial_BackendError(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) s, ps := pipe() ts := testServer(ps, 100) @@ -392,15 +448,20 @@ func TestDial_BackendError(t *testing.T) { isDialFailure, reason := GetDialFailureReason(err) if !isDialFailure { t.Errorf("Unexpected non-dial failure error: %v", err) - } else if reason != DialFailureEndpoint { + } else if reason != metrics.DialFailureEndpoint { t.Errorf("Expected DialFailureEndpoint, got %v", reason) } ts.assertPacketType(0, client.PacketType_DIAL_REQ) + + if err := metricstest.ExpectClientDialFailure(metrics.DialFailureEndpoint, 1); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. } func TestDial_Closed(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) s, ps := pipe() defer ps.Close() @@ -434,12 +495,17 @@ func TestDial_Closed(t *testing.T) { isDialFailure, reason := GetDialFailureReason(err) if !isDialFailure { t.Errorf("Unexpected non-dial failure error: %v", err) - } else if reason != DialFailureDialClosed { + } else if reason != metrics.DialFailureDialClosed { t.Errorf("Expected DialFailureDialClosed, got %v", reason) } ts.assertPacketType(0, client.PacketType_DIAL_REQ) + if err := metricstest.ExpectClientDialFailure(metrics.DialFailureDialClosed, 1); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. + select { case <-tunnel.Done(): case <-time.After(30 * time.Second): @@ -448,6 +514,14 @@ func TestDial_Closed(t *testing.T) { }() } +func TestRegisterMetrics(t *testing.T) { + Metrics.RegisterMetrics(prometheus.DefaultRegisterer) +} + +func TestLegacyRegisterMetrics(t *testing.T) { + Metrics.LegacyRegisterMetrics(prometheus.MustRegister) +} + // TODO: Move to common testing library // fakeStream implements ProxyService_ProxyClient @@ -634,3 +708,22 @@ func (s *proxyServer) handleData(pkt *client.Packet) *client.Packet { }, } } + +func assertNoClientDialFailures(t testing.TB) { + t.Helper() + if err := metricstest.ExpectClientDialFailures(nil); err != nil { + t.Errorf("Unexpected %s metric: %v", "dial_failure_total", err) + } +} + +func expectCleanShutdown(t testing.TB) { + metrics.Metrics.Reset() + currentGoRoutines := goleak.IgnoreCurrent() + t.Cleanup(func() { + goleak.VerifyNone(t, currentGoRoutines) + assertNoClientDialFailures(t) + // The tunnels gauge metric is intentionally not included here, to avoid + // brittle gauge assertions. Tests should directly verify it. + metrics.Metrics.Reset() + }) +} diff --git a/konnectivity-client/pkg/client/conn.go b/konnectivity-client/pkg/client/conn.go index f76b1e37a..14384a62c 100644 --- a/konnectivity-client/pkg/client/conn.go +++ b/konnectivity-client/pkg/client/conn.go @@ -23,6 +23,9 @@ import ( "time" "k8s.io/klog/v2" + + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) @@ -62,8 +65,11 @@ func (c *conn) Write(data []byte) (n int, err error) { klog.V(5).InfoS("[tracing] send req", "type", req.Type) + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, req.Type) err = c.stream.Send(req) if err != nil { + metrics.Metrics.ObserveStreamError(segment, err, req.Type) return 0, err } return len(data), err @@ -147,7 +153,10 @@ func (c *conn) Close() error { klog.V(5).InfoS("[tracing] send req", "type", req.Type) + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, req.Type) if err := c.stream.Send(req); err != nil { + metrics.Metrics.ObserveStreamError(segment, err, req.Type) return err } diff --git a/konnectivity-client/pkg/client/metrics/metrics.go b/konnectivity-client/pkg/client/metrics/metrics.go new file mode 100644 index 000000000..03e9d94da --- /dev/null +++ b/konnectivity-client/pkg/client/metrics/metrics.go @@ -0,0 +1,162 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" +) + +const ( + Namespace = "konnectivity_network_proxy" + Subsystem = "client" +) + +var ( + // Metrics provides access to all client metrics. The client + // application is responsible for registering (via Metrics.RegisterMetrics). + Metrics = newMetrics() +) + +// ClientMetrics includes all the metrics of the konnectivity-client. +type ClientMetrics struct { + registerOnce sync.Once + streamPackets *prometheus.CounterVec + streamErrors *prometheus.CounterVec + dialFailures *prometheus.CounterVec + clientConns *prometheus.GaugeVec +} + +type DialFailureReason string + +const ( + DialFailureUnknown DialFailureReason = "unknown" + // DialFailureTimeout indicates the hard 30 second timeout was hit. + DialFailureTimeout DialFailureReason = "timeout" + // DialFailureContext indicates that the context was cancelled or reached it's deadline before + // the dial response was returned. + DialFailureContext DialFailureReason = "context" + // DialFailureEndpoint indicates that the konnectivity-agent was unable to reach the backend endpoint. + DialFailureEndpoint DialFailureReason = "endpoint" + // DialFailureDialClosed indicates that the client received a CloseDial response, indicating the + // connection was closed before the dial could complete. + DialFailureDialClosed DialFailureReason = "dialclosed" + // DialFailureTunnelClosed indicates that the client connection was closed before the dial could + // complete. + DialFailureTunnelClosed DialFailureReason = "tunnelclosed" +) + +type ClientConnectionStatus string + +const ( + // The connection is created but has not yet been dialed. + ClientConnectionStatusCreated ClientConnectionStatus = "created" + // The connection is pending dial response. + ClientConnectionStatusDialing ClientConnectionStatus = "dialing" + // The connection is established. + ClientConnectionStatusOk ClientConnectionStatus = "ok" + // The connection is closing. + ClientConnectionStatusClosing ClientConnectionStatus = "closing" +) + +func newMetrics() *ClientMetrics { + // The denominator (total dials started) for both + // dial_failure_total and dial_duration_seconds is the + // stream_packets_total (common metric), where segment is + // "from_client" and packet_type is "DIAL_REQ". + dialFailures := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "dial_failure_total", + Help: "Number of dial failures observed, by reason (example: remote endpoint error)", + }, + []string{ + "reason", + }, + ) + clientConns := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "client_connections", + Help: "Number of open client connections, by status (Example: dialing)", + }, + []string{ + "status", + }, + ) + return &ClientMetrics{ + streamPackets: commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem), + streamErrors: commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem), + dialFailures: dialFailures, + clientConns: clientConns, + } +} + +// RegisterMetrics registers all metrics with the client application. +func (c *ClientMetrics) RegisterMetrics(r prometheus.Registerer) { + c.registerOnce.Do(func() { + r.MustRegister(c.streamPackets) + r.MustRegister(c.streamErrors) + r.MustRegister(c.dialFailures) + r.MustRegister(c.clientConns) + }) +} + +// LegacyRegisterMetrics registers all metrics via MustRegister func. +// TODO: remove this once https://github.com/kubernetes/kubernetes/pull/114293 is available. +func (c *ClientMetrics) LegacyRegisterMetrics(mustRegisterFn func(...prometheus.Collector)) { + c.registerOnce.Do(func() { + mustRegisterFn(c.streamPackets) + mustRegisterFn(c.streamErrors) + mustRegisterFn(c.dialFailures) + mustRegisterFn(c.clientConns) + }) +} + +// Reset resets the metrics. +func (c *ClientMetrics) Reset() { + c.streamPackets.Reset() + c.streamErrors.Reset() + c.dialFailures.Reset() + c.clientConns.Reset() +} + +func (c *ClientMetrics) ObserveDialFailure(reason DialFailureReason) { + c.dialFailures.WithLabelValues(string(reason)).Inc() +} + +func (c *ClientMetrics) GetClientConnectionsMetric() *prometheus.GaugeVec { + return c.clientConns +} + +func (c *ClientMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { + commonmetrics.ObservePacket(c.streamPackets, segment, packetType) +} + +func (c *ClientMetrics) ObserveStreamErrorNoPacket(segment commonmetrics.Segment, err error) { + commonmetrics.ObserveStreamErrorNoPacket(c.streamErrors, segment, err) +} + +func (c *ClientMetrics) ObserveStreamError(segment commonmetrics.Segment, err error, packetType client.PacketType) { + commonmetrics.ObserveStreamError(c.streamErrors, segment, err, packetType) +} diff --git a/konnectivity-client/pkg/common/metrics/metrics.go b/konnectivity-client/pkg/common/metrics/metrics.go new file mode 100644 index 000000000..e8619f472 --- /dev/null +++ b/konnectivity-client/pkg/common/metrics/metrics.go @@ -0,0 +1,78 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package metrics provides metric definitions and helpers used +// across konnectivity client, server, and agent. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/status" + + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" +) + +// Segment identifies one of four tunnel segments (e.g. from server to agent). +type Segment string + +const ( + // SegmentFromClient indicates a packet from client to server. + SegmentFromClient Segment = "from_client" + // SegmentToClient indicates a packet from server to client. + SegmentToClient Segment = "to_client" + // SegmentFromAgent indicates a packet from agent to server. + SegmentFromAgent Segment = "from_agent" + // SegmentToAgent indicates a packet from server to agent. + SegmentToAgent Segment = "to_agent" +) + +func MakeStreamPacketsTotalMetric(namespace, subsystem string) *prometheus.CounterVec { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "stream_packets_total", + Help: "Count of packets processed, by segment and packet type (example: from_client, DIAL_REQ)", + }, + []string{"segment", "packet_type"}, + ) +} + +func MakeStreamErrorsTotalMetric(namespace, subsystem string) *prometheus.CounterVec { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "stream_errors_total", + Help: "Count of gRPC stream errors, by segment, grpc Code, packet type. (example: from_agent, Code.Unavailable, DIAL_RSP)", + }, + []string{"segment", "code", "packet_type"}, + ) +} + +func ObservePacket(m *prometheus.CounterVec, segment Segment, packetType client.PacketType) { + m.WithLabelValues(string(segment), packetType.String()).Inc() +} + +func ObserveStreamErrorNoPacket(m *prometheus.CounterVec, segment Segment, err error) { + code := status.Code(err) + m.WithLabelValues(string(segment), code.String(), "Unknown").Inc() +} + +func ObserveStreamError(m *prometheus.CounterVec, segment Segment, err error, packetType client.PacketType) { + code := status.Code(err) + m.WithLabelValues(string(segment), code.String(), packetType.String()).Inc() +} diff --git a/konnectivity-client/pkg/common/metrics/testing/metrics.go b/konnectivity-client/pkg/common/metrics/testing/metrics.go new file mode 100644 index 000000000..85f146484 --- /dev/null +++ b/konnectivity-client/pkg/common/metrics/testing/metrics.go @@ -0,0 +1,67 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "strings" + + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" +) + +const ( + clientDialFailureHeader = ` +# HELP konnectivity_network_proxy_client_dial_failure_total Number of dial failures observed, by reason (example: remote endpoint error) +# TYPE konnectivity_network_proxy_client_dial_failure_total counter` + clientDialFailureSample = `konnectivity_network_proxy_client_dial_failure_total{reason="%s"} %d` + + clientConnsHeader = ` +# HELP konnectivity_network_proxy_client_client_connections Number of open client connections, by status (Example: dialing) +# TYPE konnectivity_network_proxy_client_client_connections gauge` + clientConnsSample = `konnectivity_network_proxy_client_client_connections{status="%s"} %d` +) + +func ExpectClientDialFailures(expected map[metrics.DialFailureReason]int) error { + expect := clientDialFailureHeader + "\n" + for r, v := range expected { + expect += fmt.Sprintf(clientDialFailureSample+"\n", r, v) + } + return ExpectMetric(metrics.Namespace, metrics.Subsystem, "dial_failure_total", expect) +} + +func ExpectClientDialFailure(reason metrics.DialFailureReason, count int) error { + return ExpectClientDialFailures(map[metrics.DialFailureReason]int{reason: count}) +} + +func ExpectClientConnections(expected map[metrics.ClientConnectionStatus]int) error { + expect := clientConnsHeader + "\n" + for r, v := range expected { + expect += fmt.Sprintf(clientConnsSample+"\n", r, v) + } + return ExpectMetric(metrics.Namespace, metrics.Subsystem, "client_connections", expect) +} + +func ExpectClientConnection(status metrics.ClientConnectionStatus, count int) error { + return ExpectClientConnections(map[metrics.ClientConnectionStatus]int{status: count}) +} + +func ExpectMetric(namespace, subsystem, name, expected string) error { + fqName := prometheus.BuildFQName(namespace, subsystem, name) + return promtest.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected), fqName) +} diff --git a/pkg/agent/client.go b/pkg/agent/client.go index 062462188..7063cbb24 100644 --- a/pkg/agent/client.go +++ b/pkg/agent/client.go @@ -35,6 +35,8 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/metadata" "k8s.io/klog/v2" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" "sigs.k8s.io/apiserver-network-proxy/proto/agent" @@ -97,6 +99,7 @@ type pendingDialContext struct { func (cm *connectionManager) Add(connID int64, ctx *connContext) { cm.connLock.Lock() defer cm.connLock.Unlock() + metrics.Metrics.EndpointConnectionInc() cm.connections[connID] = ctx } @@ -110,6 +113,9 @@ func (cm *connectionManager) Get(connID int64) (*connContext, bool) { func (cm *connectionManager) Delete(connID int64) { cm.connLock.Lock() defer cm.connLock.Unlock() + // Delete for a connID is called from cleanFunc, which is + // protected by cleanOnce. + metrics.Metrics.EndpointConnectionDec() delete(cm.connections, connID) } @@ -263,7 +269,8 @@ func newAgentClient(address, agentID, agentIdentifiers string, cs *ClientSet, op // Connect makes the grpc dial to the proxy server. It returns the serverID // it connects to. func (a *Client) Connect() (int, error) { - dialCtx, _ := context.WithTimeout(context.Background(), dialTimeout) + dialCtx, cancel := context.WithTimeout(context.Background(), dialTimeout) + defer cancel() conn, err := grpc.DialContext(dialCtx, a.address, a.opts...) if err != nil { return 0, err @@ -318,9 +325,12 @@ func (a *Client) Send(pkt *client.Packet) error { a.sendLock.Lock() defer a.sendLock.Unlock() + const segment = commonmetrics.SegmentFromAgent + metrics.Metrics.ObservePacket(segment, pkt.Type) err := a.stream.Send(pkt) if err != nil && err != io.EOF { - metrics.Metrics.ObserveFailure(metrics.DirectionToServer) + metrics.Metrics.ObserveServerFailureDeprecated(metrics.DirectionToServer) + metrics.Metrics.ObserveStreamError(segment, err, pkt.Type) a.cs.RemoveClient(a.serverID) } return err @@ -332,7 +342,8 @@ func (a *Client) Recv() (*client.Packet, error) { pkt, err := a.stream.Recv() if err != nil && err != io.EOF { - metrics.Metrics.ObserveFailure(metrics.DirectionFromServer) + metrics.Metrics.ObserveServerFailureDeprecated(metrics.DirectionFromServer) + metrics.Metrics.ObserveStreamErrorNoPacket(commonmetrics.SegmentToAgent, err) } return pkt, err } @@ -427,6 +438,7 @@ func (a *Client) Serve(biDirectional bool) { klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type) + metrics.Metrics.ObservePacket(commonmetrics.SegmentToAgent, pkt.Type) switch pkt.Type { case client.PacketType_DIAL_REQ: a.handleDialRequest(pkt) @@ -456,13 +468,13 @@ func (a *Client) Serve(biDirectional bool) { } func (a *Client) handleDialRequest(pkt *client.Packet) { - klog.V(4).InfoS("received DIAL_REQ", "serverID", a.serverID, "agentID", a.agentID) + dialReq := pkt.GetDialRequest() + klog.V(3).InfoS("Received DIAL_REQ", "serverID", a.serverID, "agentID", a.agentID, "dialID", dialReq.Random, "dialAddress", dialReq.Address) dialResp := &client.Packet{ Type: client.PacketType_DIAL_RSP, Payload: &client.Packet_DialResponse{DialResponse: &client.DialResponse{}}, } - dialReq := pkt.GetDialRequest() dialResp.GetDialResponse().Random = dialReq.Random // Even identifiers are used for connections from master to node network, @@ -483,7 +495,7 @@ func (a *Client) handleDialRequest(pkt *client.Packet) { klog.ErrorS(fmt.Errorf("connection is nil"), "cannot send CLOSE_RESP to nil connection") return } - klog.V(4).InfoS("close connection", "connectionID", connID) + klog.V(4).InfoS("close connection", "dialID", dialReq.Random, "connectionID", connID, "dialAddress", dialReq.Address) var closePkt *client.Packet if connID == 0 { closePkt = &client.Packet{ @@ -520,15 +532,21 @@ func (a *Client) handleDialRequest(pkt *client.Packet) { start := time.Now() conn, err := net.DialTimeout(dialReq.Protocol, dialReq.Address, dialTimeout) if err != nil { - klog.ErrorS(err, "error dialing backend", "dialID", dialReq.Random) + reason := metrics.DialFailureUnknown + if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + reason = metrics.DialFailureTimeout + } + metrics.Metrics.ObserveDialFailure(reason) + klog.ErrorS(err, "error dialing backend", "dialID", dialReq.Random, "connectionID", connID, "dialAddress", dialReq.Address) dialResp.GetDialResponse().Error = err.Error() if err := a.Send(dialResp); err != nil { - klog.ErrorS(err, "could not send dialResp") + klog.ErrorS(err, "could not send dialResp", "dialID", dialReq.Random, "connectionID", connID) } // Cannot invoke clean up as we have no conn yet. return } metrics.Metrics.ObserveDialLatency(time.Since(start)) + klog.V(3).InfoS("Endpoint connection established", "dialID", dialReq.Random, "connectionID", connID, "dialAddress", dialReq.Address) connCtx.conn = conn a.connManager.Add(connID, connCtx) dialResp.GetDialResponse().ConnectID = connID @@ -541,7 +559,7 @@ func (a *Client) handleDialRequest(pkt *client.Packet) { "dialAddress", dialReq.Address, ) if err := a.Send(dialResp); err != nil { - klog.ErrorS(err, "could not send dialResp") + klog.ErrorS(err, "could not send dialResp", "dialID", dialReq.Random) // clean-up is normally called from remoteToProxy which we will never invoke. // So we are invoking it here to force the clean-up to occur. // However, cleanup will block until dialDone is closed. @@ -549,7 +567,6 @@ func (a *Client) handleDialRequest(pkt *client.Packet) { go runpprof.Do(context.Background(), labels, func(context.Context) { connCtx.cleanup() }) return } - klog.V(3).InfoS("Proxying new connection", "connectionID", connID) go runpprof.Do(context.Background(), labels, func(context.Context) { a.agentToProxy(connID, connCtx) }) go runpprof.Do(context.Background(), labels, func(context.Context) { a.proxyToAgent(connID, connCtx) }) }) @@ -681,7 +698,7 @@ func (a *Client) agentToProxy(connID int64, ctx *connContext) { if panicInfo := recover(); panicInfo != nil { klog.V(2).InfoS("Exiting remoteToProxy with recovery", "panicInfo", panicInfo, "connectionID", connID) } else { - klog.V(3).InfoS("Exiting remoteToProxy", "connectionID", connID) + klog.V(4).InfoS("Exiting remoteToProxy", "connectionID", connID) } }() defer ctx.cleanup() @@ -723,7 +740,7 @@ func (a *Client) proxyToAgent(connID int64, ctx *connContext) { if panicInfo := recover(); panicInfo != nil { klog.V(2).InfoS("Exiting proxyToRemote with recovery", "panicInfo", panicInfo, "connectionID", connID) } else { - klog.V(3).InfoS("Exiting proxyToRemote", "connectionID", connID) + klog.V(4).InfoS("Exiting proxyToRemote", "connectionID", connID) } }() // Not safe to call cleanup here, as cleanup() closes the dataCh diff --git a/pkg/agent/client_test.go b/pkg/agent/client_test.go index e04c97ea4..621a2da9c 100644 --- a/pkg/agent/client_test.go +++ b/pkg/agent/client_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package agent import ( diff --git a/pkg/agent/clientset.go b/pkg/agent/clientset.go index 842b611a7..40f819562 100644 --- a/pkg/agent/clientset.go +++ b/pkg/agent/clientset.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc/connectivity" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" "sigs.k8s.io/apiserver-network-proxy/pkg/features" ) @@ -112,6 +113,7 @@ func (cs *ClientSet) addClientLocked(serverID string, c *Client) error { } cs.clients[serverID] = c cs.serverIDs = append(cs.serverIDs, serverID) + metrics.Metrics.SetServerConnectionsCount(len(cs.clients)) return nil } @@ -137,6 +139,7 @@ func (cs *ClientSet) RemoveClient(serverID string) { } } delete(cs.clients, serverID) + metrics.Metrics.SetServerConnectionsCount(len(cs.clients)) } type ClientSetConfig struct { diff --git a/pkg/agent/metrics/metrics.go b/pkg/agent/metrics/metrics.go index d12b29ed9..7a7ac19b8 100644 --- a/pkg/agent/metrics/metrics.go +++ b/pkg/agent/metrics/metrics.go @@ -20,13 +20,16 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) type Direction string const ( - namespace = "konnectivity_network_proxy" - subsystem = "agent" + Namespace = "konnectivity_network_proxy" + Subsystem = "agent" // DirectionToServer indicates that the agent attempts to send a packet // to the proxy server. @@ -46,49 +49,140 @@ var ( // AgentMetrics includes all the metrics of the proxy agent. type AgentMetrics struct { - latencies *prometheus.HistogramVec - failures *prometheus.CounterVec + dialLatencies *prometheus.HistogramVec + serverFailures *prometheus.CounterVec + dialFailures *prometheus.CounterVec + serverConnections *prometheus.GaugeVec + endpointConnections *prometheus.GaugeVec + streamPackets *prometheus.CounterVec + streamErrors *prometheus.CounterVec } // newAgentMetrics create a new AgentMetrics, configured with default metric names. func newAgentMetrics() *AgentMetrics { - latencies := prometheus.NewHistogramVec( + dialLatencies := prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, + Namespace: Namespace, + Subsystem: Subsystem, Name: "dial_duration_seconds", Help: "Latency of dial to the remote endpoint in seconds", Buckets: latencyBuckets, }, []string{}, ) - failures := prometheus.NewCounterVec( + serverFailures := prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, + Namespace: Namespace, + Subsystem: Subsystem, Name: "server_connection_failure_count", - Help: "Count of failures to send to or receive from the proxy server, labeled by the direction (from_server or to_server)", + Help: "Count of failures to send to or receive from the proxy server, labeled by the direction (from_server or to_server). DEPRECATED, please use stream_events_error_total", }, []string{"direction"}, ) - prometheus.MustRegister(failures) - prometheus.MustRegister(latencies) - return &AgentMetrics{failures: failures, latencies: latencies} + dialFailures := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "endpoint_dial_failure_total", + Help: "Number of failures dialing the remote endpoint, by reason (example: timeout).", + }, + []string{"reason"}, + ) + serverConnections := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "open_server_connections", + Help: "Current number of open server connections.", + }, + []string{}, + ) + endpointConnections := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "open_endpoint_connections", + Help: "Current number of open endpoint connections.", + }, + []string{}, + ) + streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem) + streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem) + prometheus.MustRegister(dialLatencies) + prometheus.MustRegister(serverFailures) + prometheus.MustRegister(dialFailures) + prometheus.MustRegister(serverConnections) + prometheus.MustRegister(endpointConnections) + prometheus.MustRegister(streamPackets) + prometheus.MustRegister(streamErrors) + return &AgentMetrics{ + dialLatencies: dialLatencies, + serverFailures: serverFailures, + dialFailures: dialFailures, + serverConnections: serverConnections, + endpointConnections: endpointConnections, + streamPackets: streamPackets, + streamErrors: streamErrors, + } + } // Reset resets the metrics. func (a *AgentMetrics) Reset() { - a.failures.Reset() - a.latencies.Reset() + a.dialLatencies.Reset() + a.serverFailures.Reset() + a.dialFailures.Reset() + a.serverConnections.Reset() + a.endpointConnections.Reset() + a.streamPackets.Reset() + a.streamErrors.Reset() } -// ObserveFailure records a failure to send to or receive from the proxy +// ObserveServerFailure records a failure to send to or receive from the proxy // server, labeled by the direction. -func (a *AgentMetrics) ObserveFailure(direction Direction) { - a.failures.WithLabelValues(string(direction)).Inc() +func (a *AgentMetrics) ObserveServerFailureDeprecated(direction Direction) { + a.serverFailures.WithLabelValues(string(direction)).Inc() } +type DialFailureReason string + +const ( + DialFailureTimeout DialFailureReason = "timeout" + DialFailureUnknown DialFailureReason = "unknown" +) + // ObserveDialLatency records the latency of dial to the remote endpoint. func (a *AgentMetrics) ObserveDialLatency(elapsed time.Duration) { - a.latencies.WithLabelValues().Observe(elapsed.Seconds()) + a.dialLatencies.WithLabelValues().Observe(elapsed.Seconds()) +} + +// ObserveDialFailure records a remote endpoint dial failure. +func (a *AgentMetrics) ObserveDialFailure(reason DialFailureReason) { + a.dialFailures.WithLabelValues(string(reason)).Inc() +} + +func (a *AgentMetrics) SetServerConnectionsCount(count int) { + a.serverConnections.WithLabelValues().Set(float64(count)) +} + +// EndpointConnectionInc increments a new endpoint connection. +func (a *AgentMetrics) EndpointConnectionInc() { + a.endpointConnections.WithLabelValues().Inc() +} + +// EndpointConnectionDec decrements a finished endpoint connection. +func (a *AgentMetrics) EndpointConnectionDec() { + a.endpointConnections.WithLabelValues().Dec() +} + +func (a *AgentMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { + commonmetrics.ObservePacket(a.streamPackets, segment, packetType) +} + +func (a *AgentMetrics) ObserveStreamErrorNoPacket(segment commonmetrics.Segment, err error) { + commonmetrics.ObserveStreamErrorNoPacket(a.streamErrors, segment, err) +} + +func (a *AgentMetrics) ObserveStreamError(segment commonmetrics.Segment, err error, packetType client.PacketType) { + commonmetrics.ObserveStreamError(a.streamErrors, segment, err, packetType) } diff --git a/pkg/server/backend_manager.go b/pkg/server/backend_manager.go index 6fd92026b..15e5c97cc 100644 --- a/pkg/server/backend_manager.go +++ b/pkg/server/backend_manager.go @@ -25,6 +25,8 @@ import ( "time" "k8s.io/klog/v2" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent" "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" @@ -85,7 +87,13 @@ type backend struct { func (b *backend) Send(p *client.Packet) error { b.mu.Lock() defer b.mu.Unlock() - return b.conn.Send(p) + const segment = commonmetrics.SegmentToAgent + metrics.Metrics.ObservePacket(segment, p.Type) + err := b.conn.Send(p) + if err != nil { + metrics.Metrics.ObserveStreamError(segment, err, p.Type) + } + return err } func (b *backend) Context() context.Context { @@ -188,7 +196,7 @@ func (s *DefaultBackendStorage) AddBackend(identifier string, idType pkgagent.Id klog.V(4).InfoS("fail to add backend", "backend", identifier, "error", &ErrWrongIDType{idType, s.idTypes}) return nil } - klog.V(2).InfoS("Register backend for agent", "connection", conn, "agentID", identifier) + klog.V(5).InfoS("Register backend for agent", "connection", conn, "agentID", identifier) s.mu.Lock() defer s.mu.Unlock() _, ok := s.backends[identifier] @@ -218,7 +226,7 @@ func (s *DefaultBackendStorage) RemoveBackend(identifier string, idType pkgagent klog.ErrorS(&ErrWrongIDType{idType, s.idTypes}, "fail to remove backend") return } - klog.V(2).InfoS("Remove connection for agent", "connection", conn, "identifier", identifier) + klog.V(5).InfoS("Remove connection for agent", "connection", conn, "identifier", identifier) s.mu.Lock() defer s.mu.Unlock() backends, ok := s.backends[identifier] @@ -299,7 +307,7 @@ func (s *DefaultBackendStorage) GetRandomBackend() (Backend, error) { return nil, &ErrNotFound{} } agentID := s.agentIDs[s.random.Intn(len(s.agentIDs))] - klog.V(4).InfoS("Pick agent as backend", "agentID", agentID) + klog.V(5).InfoS("Pick agent as backend", "agentID", agentID) // always return the first connection to an agent, because the agent // will close later connections if there are multiple. return s.backends[agentID][0], nil diff --git a/pkg/server/metrics/metrics.go b/pkg/server/metrics/metrics.go index 6209071df..fde6b4ded 100644 --- a/pkg/server/metrics/metrics.go +++ b/pkg/server/metrics/metrics.go @@ -20,6 +20,9 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) const ( @@ -51,19 +54,21 @@ var ( // ServerMetrics includes all the metrics of the proxy server. type ServerMetrics struct { - latencies *prometheus.HistogramVec + endpointLatencies *prometheus.HistogramVec frontendLatencies *prometheus.HistogramVec - connections *prometheus.GaugeVec + grpcConnections *prometheus.GaugeVec httpConnections prometheus.Gauge backend *prometheus.GaugeVec pendingDials *prometheus.GaugeVec fullRecvChannels *prometheus.GaugeVec dialFailures *prometheus.CounterVec + streamPackets *prometheus.CounterVec + streamErrors *prometheus.CounterVec } // newServerMetrics create a new ServerMetrics, configured with default metric names. func newServerMetrics() *ServerMetrics { - latencies := prometheus.NewHistogramVec( + endpointLatencies := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: Namespace, Subsystem: Subsystem, @@ -83,7 +88,7 @@ func newServerMetrics() *ServerMetrics { }, []string{}, ) - connections := prometheus.NewGaugeVec( + grpcConnections := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: Subsystem, @@ -142,77 +147,84 @@ func newServerMetrics() *ServerMetrics { "reason", }, ) - - prometheus.MustRegister(latencies) + streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem) + streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem) + prometheus.MustRegister(endpointLatencies) prometheus.MustRegister(frontendLatencies) - prometheus.MustRegister(connections) + prometheus.MustRegister(grpcConnections) prometheus.MustRegister(httpConnections) prometheus.MustRegister(backend) prometheus.MustRegister(pendingDials) prometheus.MustRegister(fullRecvChannels) prometheus.MustRegister(dialFailures) + prometheus.MustRegister(streamPackets) + prometheus.MustRegister(streamErrors) return &ServerMetrics{ - latencies: latencies, + endpointLatencies: endpointLatencies, frontendLatencies: frontendLatencies, - connections: connections, + grpcConnections: grpcConnections, httpConnections: httpConnections, backend: backend, pendingDials: pendingDials, fullRecvChannels: fullRecvChannels, dialFailures: dialFailures, + streamPackets: streamPackets, + streamErrors: streamErrors, } } // Reset resets the metrics. -func (a *ServerMetrics) Reset() { - a.latencies.Reset() - a.frontendLatencies.Reset() - a.connections.Reset() - a.backend.Reset() - a.pendingDials.Reset() - a.fullRecvChannels.Reset() - a.dialFailures.Reset() +func (s *ServerMetrics) Reset() { + s.endpointLatencies.Reset() + s.frontendLatencies.Reset() + s.grpcConnections.Reset() + s.backend.Reset() + s.pendingDials.Reset() + s.fullRecvChannels.Reset() + s.dialFailures.Reset() + s.streamPackets.Reset() + s.streamErrors.Reset() } // ObserveDialLatency records the latency of dial to the remote endpoint. -func (a *ServerMetrics) ObserveDialLatency(elapsed time.Duration) { - a.latencies.WithLabelValues().Observe(elapsed.Seconds()) +func (s *ServerMetrics) ObserveDialLatency(elapsed time.Duration) { + s.endpointLatencies.WithLabelValues().Observe(elapsed.Seconds()) } -// ObserveFrontendWriteLatency records the latency of dial to the remote endpoint. -func (a *ServerMetrics) ObserveFrontendWriteLatency(elapsed time.Duration) { - a.frontendLatencies.WithLabelValues().Observe(elapsed.Seconds()) +// ObserveFrontendWriteLatency records the latency of blocking on stream send to the client. +func (s *ServerMetrics) ObserveFrontendWriteLatency(elapsed time.Duration) { + s.frontendLatencies.WithLabelValues().Observe(elapsed.Seconds()) } // ConnectionInc increments a new grpc client connection. -func (a *ServerMetrics) ConnectionInc(serviceMethod string) { - a.connections.With(prometheus.Labels{"service_method": serviceMethod}).Inc() +func (s *ServerMetrics) ConnectionInc(serviceMethod string) { + s.grpcConnections.With(prometheus.Labels{"service_method": serviceMethod}).Inc() } // ConnectionDec decrements a finished grpc client connection. -func (a *ServerMetrics) ConnectionDec(serviceMethod string) { - a.connections.With(prometheus.Labels{"service_method": serviceMethod}).Dec() +func (s *ServerMetrics) ConnectionDec(serviceMethod string) { + s.grpcConnections.With(prometheus.Labels{"service_method": serviceMethod}).Dec() } // HTTPConnectionDec increments a new HTTP CONNECTION connection. -func (a *ServerMetrics) HTTPConnectionInc() { a.httpConnections.Inc() } +func (s *ServerMetrics) HTTPConnectionInc() { s.httpConnections.Inc() } // HTTPConnectionDec decrements a finished HTTP CONNECTION connection. -func (a *ServerMetrics) HTTPConnectionDec() { a.httpConnections.Dec() } +func (s *ServerMetrics) HTTPConnectionDec() { s.httpConnections.Dec() } // SetBackendCount sets the number of backend connection. -func (a *ServerMetrics) SetBackendCount(count int) { - a.backend.WithLabelValues().Set(float64(count)) +func (s *ServerMetrics) SetBackendCount(count int) { + s.backend.WithLabelValues().Set(float64(count)) } // SetPendingDialCount sets the number of pending dials. -func (a *ServerMetrics) SetPendingDialCount(count int) { - a.pendingDials.WithLabelValues().Set(float64(count)) +func (s *ServerMetrics) SetPendingDialCount(count int) { + s.pendingDials.WithLabelValues().Set(float64(count)) } // FullRecvChannel retrieves the metric for counting full receive channels. -func (a *ServerMetrics) FullRecvChannel(serviceMethod string) prometheus.Gauge { - return a.fullRecvChannels.With(prometheus.Labels{"service_method": serviceMethod}) +func (s *ServerMetrics) FullRecvChannel(serviceMethod string) prometheus.Gauge { + return s.fullRecvChannels.With(prometheus.Labels{"service_method": serviceMethod}) } type DialFailureReason string @@ -225,6 +237,18 @@ const ( DialFailureFrontendClose DialFailureReason = "frontend_close" // Received a DIAL_CLS from the frontend before the dial completed. ) -func (a *ServerMetrics) ObserveDialFailure(reason DialFailureReason) { - a.dialFailures.With(prometheus.Labels{"reason": string(reason)}).Inc() +func (s *ServerMetrics) ObserveDialFailure(reason DialFailureReason) { + s.dialFailures.With(prometheus.Labels{"reason": string(reason)}).Inc() +} + +func (s *ServerMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { + commonmetrics.ObservePacket(s.streamPackets, segment, packetType) +} + +func (s *ServerMetrics) ObserveStreamErrorNoPacket(segment commonmetrics.Segment, err error) { + commonmetrics.ObserveStreamErrorNoPacket(s.streamErrors, segment, err) +} + +func (s *ServerMetrics) ObserveStreamError(segment commonmetrics.Segment, err error, packetType client.PacketType) { + commonmetrics.ObserveStreamError(s.streamErrors, segment, err, packetType) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 83fbe1bab..4edf55cc6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -36,6 +36,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent" "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" @@ -66,11 +68,17 @@ const ( ) func (c *ProxyClientConnection) send(pkt *client.Packet) error { + const segment = commonmetrics.SegmentToClient + metrics.Metrics.ObservePacket(segment, pkt.Type) start := time.Now() defer metrics.Metrics.ObserveFrontendWriteLatency(time.Since(start)) if c.Mode == "grpc" { stream := c.Grpc - return stream.Send(pkt) + err := stream.Send(pkt) + if err != nil { + metrics.Metrics.ObserveStreamError(segment, err, pkt.Type) + } + return err } else if c.Mode == "http-connect" { if pkt.Type == client.PacketType_CLOSE_RSP { return c.CloseHTTP() @@ -278,7 +286,6 @@ func (s *ProxyServer) removeBackend(agentID string, conn agent.AgentService_Conn } func (s *ProxyServer) addFrontend(agentID string, connID int64, p *ProxyClientConnection) { - klog.V(2).InfoS("Register frontend for agent", "frontend", p, "agentID", agentID, "connectionID", connID) s.fmu.Lock() defer s.fmu.Unlock() if _, ok := s.frontends[agentID]; !ok { @@ -299,7 +306,6 @@ func (s *ProxyServer) removeFrontend(agentID string, connID int64) { klog.V(2).InfoS("Cannot find connection for agent in the frontends", "connectionID", connID, "agentID", agentID) return } - klog.V(2).InfoS("Remove frontend for agent", "frontend", conns[connID], "agentID", agentID, "connectionID", connID) delete(s.frontends[agentID], connID) if len(s.frontends[agentID]) == 0 { delete(s.frontends, agentID) @@ -377,7 +383,7 @@ func (s *ProxyServer) Proxy(stream client.ProxyService_ProxyServer) error { return fmt.Errorf("failed to get context") } userAgent := md.Get(header.UserAgent) - klog.V(2).InfoS("Proxy request from client", "userAgent", userAgent, "serverID", s.serverID) + klog.V(5).InfoS("Proxy request from client", "userAgent", userAgent, "serverID", s.serverID) recvCh := make(chan *client.Packet, xfrChannelSize) stopCh := make(chan error) @@ -389,7 +395,6 @@ func (s *ProxyServer) Proxy(stream client.ProxyService_ProxyServer) error { go runpprof.Do(context.Background(), labels, func(context.Context) { s.serveRecvFrontend(stream, recvCh) }) defer func() { - klog.V(2).InfoS("Receive channel from frontend is stopping", "userAgent", userAgent) close(recvCh) }() @@ -403,11 +408,14 @@ func (s *ProxyServer) readFrontendToChannel(stream client.ProxyService_ProxyServ for { in, err := stream.Recv() if err == io.EOF { + // TODO: Log an error if the frontend connection stops without first closing any open connections. klog.V(2).InfoS("Receive stream from frontend closed", "userAgent", userAgent) close(stopCh) return } + const segment = commonmetrics.SegmentFromClient if err != nil { + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) if status.Code(err) == codes.Canceled { klog.V(2).InfoS("Stream read from frontend cancelled", "userAgent", userAgent) } else { @@ -417,6 +425,7 @@ func (s *ProxyServer) readFrontendToChannel(stream client.ProxyService_ProxyServ close(stopCh) return } + metrics.Metrics.ObservePacket(segment, in.Type) select { case recvCh <- in: // Send didn't block, carry on. @@ -462,7 +471,10 @@ func (s *ProxyServer) serveRecvFrontend(stream client.ProxyService_ProxyServer, }, }, } + const segment = commonmetrics.SegmentToClient + metrics.Metrics.ObservePacket(segment, resp.Type) if err := stream.Send(resp); err != nil { + metrics.Metrics.ObserveStreamError(segment, err, resp.Type) klog.V(5).InfoS("Failed to send DIAL_RSP for no backend", "error", err, "dialID", random) } // The Dial is failing; no reason to keep this goroutine. @@ -539,6 +551,7 @@ func (s *ProxyServer) serveRecvFrontend(stream client.ProxyService_ProxyServer, } } + // TODO: Log an error if the frontend stream is closed with an established tunnel still open. klog.V(5).InfoS("Close streaming", "connectionID", firstConnID) if backend == nil { @@ -594,7 +607,7 @@ func getAgentIdentifiers(stream agent.AgentService_ConnectServer) (pkgagent.Iden return agentIdentifiers, nil } -func (s *ProxyServer) validateAuthToken(ctx context.Context, token string) error { +func (s *ProxyServer) validateAuthToken(ctx context.Context, token string) (username string, err error) { trReq := &authv1.TokenReview{ Spec: authv1.TokenReviewSpec{ Token: token, @@ -603,38 +616,39 @@ func (s *ProxyServer) validateAuthToken(ctx context.Context, token string) error } r, err := s.AgentAuthenticationOptions.KubernetesClient.AuthenticationV1().TokenReviews().Create(ctx, trReq, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("failed to authenticate request. err:%v", err) + return "", fmt.Errorf("Failed to authenticate request. err:%v", err) } if r.Status.Error != "" { - return fmt.Errorf("lookup failed: %s", r.Status.Error) + return "", fmt.Errorf("lookup failed: %s", r.Status.Error) } if !r.Status.Authenticated { - return fmt.Errorf("lookup failed: service account jwt not valid") + return "", fmt.Errorf("lookup failed: service account jwt not valid") } // The username is of format: system:serviceaccount:(NAMESPACE):(SERVICEACCOUNT) - parts := strings.Split(r.Status.User.Username, ":") + username = r.Status.User.Username + parts := strings.Split(username, ":") if len(parts) != 4 { - return fmt.Errorf("lookup failed: unexpected username format") + return "", fmt.Errorf("lookup failed: unexpected username format") } // Validate the user that comes back from token review is a service account if parts[0] != "system" || parts[1] != "serviceaccount" { - return fmt.Errorf("lookup failed: username returned is not a service account") + return "", fmt.Errorf("lookup failed: username returned is not a service account") } ns := parts[2] sa := parts[3] if s.AgentAuthenticationOptions.AgentNamespace != ns { - return fmt.Errorf("lookup failed: incoming request from %q namespace. Expected %q", ns, s.AgentAuthenticationOptions.AgentNamespace) + return "", fmt.Errorf("lookup failed: incoming request from %q namespace. Expected %q", ns, s.AgentAuthenticationOptions.AgentNamespace) } if s.AgentAuthenticationOptions.AgentServiceAccount != sa { - return fmt.Errorf("lookup failed: incoming request from %q service account. Expected %q", sa, s.AgentAuthenticationOptions.AgentServiceAccount) + return "", fmt.Errorf("lookup failed: incoming request from %q service account. Expected %q", sa, s.AgentAuthenticationOptions.AgentServiceAccount) } - return nil + return username, nil } func (s *ProxyServer) authenticateAgentViaToken(ctx context.Context) error { @@ -656,11 +670,12 @@ func (s *ProxyServer) authenticateAgentViaToken(ctx context.Context) error { return fmt.Errorf("received token does not have %q prefix", header.AuthenticationTokenContextSchemePrefix) } - if err := s.validateAuthToken(ctx, strings.TrimPrefix(authContext[0], header.AuthenticationTokenContextSchemePrefix)); err != nil { - return fmt.Errorf("failed to validate authentication token, err:%v", err) + username, err := s.validateAuthToken(ctx, strings.TrimPrefix(authContext[0], header.AuthenticationTokenContextSchemePrefix)) + if err != nil { + return fmt.Errorf("Failed to validate authentication token, err:%v", err) } - klog.V(2).Infoln("Client successfully authenticated via token") + klog.V(5).InfoS("Agent successfully authenticated via token", "username", username) return nil } @@ -674,7 +689,7 @@ func (s *ProxyServer) Connect(stream agent.AgentService_ConnectServer) error { return err } - klog.V(2).InfoS("Connect request from agent", "agentID", agentID, "serverID", s.serverID) + klog.V(5).InfoS("Connect request from agent", "agentID", agentID, "serverID", s.serverID) labels := runpprof.Labels( "serverCount", strconv.Itoa(s.serverCount), "agentID", agentID, @@ -695,20 +710,20 @@ func (s *ProxyServer) Connect(stream agent.AgentService_ConnectServer) error { return err } - var nilBackend Backend = nil + klog.V(2).InfoS("Agent connected", "agentID", agentID, "serverID", s.serverID) backend := s.addBackend(agentID, stream) + // we can't compare `backend` to nil equality because it is interface variable + var nilBackend Backend // = nil if backend == nilBackend { klog.V(2).InfoS("Unable to add backend", "agentID", agentID, "serverID", s.serverID) return fmt.Errorf("no backend added") } - // we can't compare `backend` to nil equality because it is interface variable defer s.removeBackend(agentID, stream) recvCh := make(chan *client.Packet, xfrChannelSize) go runpprof.Do(context.Background(), labels, func(context.Context) { s.serveRecvBackend(backend, stream, agentID, recvCh) }) defer func() { - klog.V(2).InfoS("Receive channel from agent is stopping", "agentID", agentID) close(recvCh) }() @@ -726,12 +741,15 @@ func (s *ProxyServer) readBackendToChannel(stream agent.AgentService_ConnectServ close(stopCh) return } + const segment = commonmetrics.SegmentFromAgent if err != nil { + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) klog.ErrorS(err, "Receive stream from agent read failure") stopCh <- err close(stopCh) return } + metrics.Metrics.ObservePacket(segment, in.Type) select { case recvCh <- in: // Send didn't block, carry on. @@ -904,6 +922,7 @@ func (s *ProxyServer) serveRecvBackend(backend Backend, stream agent.AgentServic } klog.V(5).InfoS("Close backend of agent", "backend", stream, "agentID", agentID) } + func (s *ProxyServer) sendCloseRequest(stream agent.AgentService_ConnectServer, connectID int64, random int64, failMsg string) { pkt := &client.Packet{ Type: client.PacketType_CLOSE_REQ, @@ -913,7 +932,10 @@ func (s *ProxyServer) sendCloseRequest(stream agent.AgentService_ConnectServer, }, }, } + const segment = commonmetrics.SegmentToAgent + metrics.Metrics.ObservePacket(segment, pkt.Type) if err := stream.Send(pkt); err != nil { + metrics.Metrics.ObserveStreamError(segment, err, pkt.Type) klog.V(5).ErrorS(err, failMsg, "dialID", random, "agentID", agentID, "connectionID", connectID) } } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 0c569af98..5695b71d5 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -242,12 +242,12 @@ func (f testStream) Context() context.Context { return ctx } -func (t testStream) Send(packet *client.Packet) error { +func (f testStream) Send(packet *client.Packet) error { return nil } -func (t testStream) Recv() (*client.Packet, error) { - v, ok := <-t.ch +func (f testStream) Recv() (*client.Packet, error) { + v, ok := <-f.ch if !ok { return nil, io.EOF } @@ -265,7 +265,7 @@ func TestNodeToControlPlane(t *testing.T) { &AgentTokenAuthenticationOptions{}) // start a controlplane server - lis, err := net.Listen("tcp", ":0") + lis, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { panic(err) } @@ -345,7 +345,7 @@ func packetDialReq(addr string) *client.Packet { DialRequest: &client.DialRequest{ Protocol: "tcp", Address: addr, - Random: rand.Int63(), + Random: rand.Int63(), /* #nosec G404 */ }, }, } diff --git a/pkg/testing/metrics/metrics.go b/pkg/testing/metrics/metrics.go index cabc9e98e..a7501390b 100644 --- a/pkg/testing/metrics/metrics.go +++ b/pkg/testing/metrics/metrics.go @@ -22,29 +22,47 @@ import ( "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" + agent "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" server "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" ) const ( - dialFailureHeader = ` + serverDialFailureHeader = ` # HELP konnectivity_network_proxy_server_dial_failure_count Number of dial failures observed. Multiple failures can occur for a single dial request. # TYPE konnectivity_network_proxy_server_dial_failure_count counter` - dialFailureSample = `konnectivity_network_proxy_server_dial_failure_count{reason="%s"} %d` + serverDialFailureSample = `konnectivity_network_proxy_server_dial_failure_count{reason="%s"} %d` + + agentDialFailureHeader = ` +# HELP konnectivity_network_proxy_agent_endpoint_dial_failure_total Number of failures dialing the remote endpoint, by reason (example: timeout). +# TYPE konnectivity_network_proxy_agent_endpoint_dial_failure_total counter` + agentDialFailureSample = `konnectivity_network_proxy_agent_endpoint_dial_failure_total{reason="%s"} %d` ) -func ExpectDialFailures(expected map[server.DialFailureReason]int) error { - expect := dialFailureHeader + "\n" +func ExpectServerDialFailures(expected map[server.DialFailureReason]int) error { + expect := serverDialFailureHeader + "\n" + for r, v := range expected { + expect += fmt.Sprintf(serverDialFailureSample+"\n", r, v) + } + return ExpectMetric(server.Namespace, server.Subsystem, server.DialFailuresMetric, expect) +} + +func ExpectServerDialFailure(reason server.DialFailureReason, count int) error { + return ExpectServerDialFailures(map[server.DialFailureReason]int{reason: count}) +} + +func ExpectAgentDialFailures(expected map[agent.DialFailureReason]int) error { + expect := agentDialFailureHeader + "\n" for r, v := range expected { - expect += fmt.Sprintf(dialFailureSample+"\n", r, v) + expect += fmt.Sprintf(agentDialFailureSample+"\n", r, v) } - return ExpectMetric(server.Subsystem, server.DialFailuresMetric, expect) + return ExpectMetric(agent.Namespace, agent.Subsystem, "endpoint_dial_failure_total", expect) } -func ExpectDialFailure(reason server.DialFailureReason, count int) error { - return ExpectDialFailures(map[server.DialFailureReason]int{reason: count}) +func ExpectAgentDialFailure(reason agent.DialFailureReason, count int) error { + return ExpectAgentDialFailures(map[agent.DialFailureReason]int{reason: count}) } -func ExpectMetric(subsystem, name, expected string) error { - fqName := prometheus.BuildFQName(server.Namespace, subsystem, name) +func ExpectMetric(namespace, subsystem, name, expected string) error { + fqName := prometheus.BuildFQName(namespace, subsystem, name) return promtest.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected), fqName) } diff --git a/tests/agent_disconnect_test.go b/tests/agent_disconnect_test.go index 1453359a7..09bec5ff3 100644 --- a/tests/agent_disconnect_test.go +++ b/tests/agent_disconnect_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/benchmarks_test.go b/tests/benchmarks_test.go new file mode 100644 index 000000000..3e06b406a --- /dev/null +++ b/tests/benchmarks_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/grpc" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" +) + +func BenchmarkLargeResponse_GRPC(b *testing.B) { + b.StopTimer() + + expectCleanShutdown(b) + + ctx := context.Background() + const length = 1 << 20 // 1M + const chunks = 10 + server := httptest.NewServer(newSizedServer(length, chunks)) + defer server.Close() + + stopCh := make(chan struct{}) + defer close(stopCh) + + proxy, cleanup, err := runGRPCProxyServer() + if err != nil { + b.Fatal(err) + } + defer cleanup() + + clientset := runAgent(proxy.agent, stopCh) + waitForConnectedServerCount(b, 1, clientset) + + req, err := http.NewRequest("GET", server.URL, nil) + if err != nil { + b.Fatal(err) + } + req.Close = true + + for n := 0; n < b.N; n++ { + // run test client + tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, proxy.front, grpc.WithInsecure()) + if err != nil { + b.Fatal(err) + } + + c := &http.Client{ + Transport: &http.Transport{ + DialContext: tunnel.DialContext, + }, + } + + r, err := c.Do(req) + if err != nil { + b.Fatal(err) + } + + b.StartTimer() // BEGIN CRITICAL SECTION + + size, err := io.Copy(io.Discard, r.Body) + if err != nil { + b.Fatal(err) + } + r.Body.Close() + + b.StopTimer() // END CRITICAL SECTION + + if size != length*chunks { + b.Fatalf("expect data length %d; got %d", length*chunks, size) + } + } +} + +func BenchmarkLargeRequest_GRPC(b *testing.B) { + b.StopTimer() + + expectCleanShutdown(b) + + const length = 10 << 20 // 10M + + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + size, err := io.Copy(io.Discard, req.Body) + if err != nil { + b.Fatal(err) + } + if size != length { + b.Fatalf("Expected data length %d; got %d", length, size) + } + req.Body.Close() + + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + stopCh := make(chan struct{}) + defer close(stopCh) + + proxy, cleanup, err := runGRPCProxyServer() + if err != nil { + b.Fatal(err) + } + defer cleanup() + + clientset := runAgent(proxy.agent, stopCh) + waitForConnectedServerCount(b, 1, clientset) + + bodyBytes := make([]byte, length) + body := bytes.NewReader(bodyBytes) + req, err := http.NewRequest("POST", server.URL, body) + if err != nil { + b.Fatal(err) + } + req.Close = true + for n := 0; n < b.N; n++ { + // run test client + tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, proxy.front, grpc.WithInsecure()) + if err != nil { + b.Fatal(err) + } + + c := &http.Client{ + Transport: &http.Transport{ + DialContext: tunnel.DialContext, + }, + } + body.Reset(bodyBytes) // We're reusing the request, so make sure to reset the body reader. + + b.StartTimer() // BEGIN CRITICAL SECTION + + if _, err := c.Do(req); err != nil { + b.Fatal(err) + } + + b.StopTimer() // END CRITICAL SECTION + } +} diff --git a/tests/concurrent_client_request_test.go b/tests/concurrent_client_request_test.go index bd1fafdd8..5f2f89714 100644 --- a/tests/concurrent_client_request_test.go +++ b/tests/concurrent_client_request_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/concurrent_test.go b/tests/concurrent_test.go index 8efb0fdde..d6d906c39 100644 --- a/tests/concurrent_test.go +++ b/tests/concurrent_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/custom_alpn_test.go b/tests/custom_alpn_test.go index 0ce40e3de..9f96e475e 100644 --- a/tests/custom_alpn_test.go +++ b/tests/custom_alpn_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/ha_proxy_server_test.go b/tests/ha_proxy_server_test.go index 2e703e0af..bb5ea0f91 100644 --- a/tests/ha_proxy_server_test.go +++ b/tests/ha_proxy_server_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/main_test.go b/tests/main_test.go index a9d652dd2..efc51a166 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -1,16 +1,36 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( "flag" "testing" + "github.com/prometheus/client_golang/prometheus" "k8s.io/klog/v2" + + metricsclient "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" ) func TestMain(m *testing.M) { fs := flag.NewFlagSet("mock-flags", flag.PanicOnError) klog.InitFlags(fs) fs.Set("v", "1") // Set klog verbosity. + metricsclient.Metrics.RegisterMetrics(prometheus.DefaultRegisterer) m.Run() } diff --git a/tests/proxy_test.go b/tests/proxy_test.go index 10e7e5246..99055ecda 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( @@ -20,10 +36,14 @@ import ( "google.golang.org/grpc/metadata" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + metricsclient "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + clientmetricstest "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics/testing" clientproto "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" "sigs.k8s.io/apiserver-network-proxy/pkg/agent" + metricsagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" "sigs.k8s.io/apiserver-network-proxy/pkg/server" - "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" + metricsserver "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" metricstest "sigs.k8s.io/apiserver-network-proxy/pkg/testing/metrics" agentproto "sigs.k8s.io/apiserver-network-proxy/proto/agent" "sigs.k8s.io/apiserver-network-proxy/proto/header" @@ -169,10 +189,13 @@ func TestProxyHandleDialError_GRPC(t *testing.T) { t.Errorf("Unexpected error: %v", err) } - if err := metricstest.ExpectDialFailure(metrics.DialFailureErrorResponse, 1); err != nil { + if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureErrorResponse, 1); err != nil { + t.Error(err) + } + if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureUnknown, 1); err != nil { t.Error(err) } - metrics.Metrics.Reset() // For clean shutdown. + resetAllMetrics() // For clean shutdown. } func TestProxyHandle_DoneContext_GRPC(t *testing.T) { @@ -304,7 +327,7 @@ func TestProxyDial_RequestCancelled_GRPC(t *testing.T) { _, err = tunnel.DialContext(ctx, "tcp", blackhole) if err == nil { t.Error("Expected error when context is cancelled, did not receive error") - } else if _, reason := client.GetDialFailureReason(err); reason != client.DialFailureContext { + } else if _, reason := client.GetDialFailureReason(err); reason != metricsclient.DialFailureContext { t.Errorf("Unexpected error: %v", err) } @@ -315,10 +338,13 @@ func TestProxyDial_RequestCancelled_GRPC(t *testing.T) { } }() - if err := metricstest.ExpectDialFailure(metrics.DialFailureFrontendClose, 1); err != nil { + if err := clientmetricstest.ExpectClientDialFailure(metrics.DialFailureContext, 1); err != nil { t.Error(err) } - metrics.Metrics.Reset() // For clean shutdown. + if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureFrontendClose, 1); err != nil { + t.Error(err) + } + resetAllMetrics() // For clean shutdown. } func TestProxyDial_AgentTimeout_GRPC(t *testing.T) { @@ -349,14 +375,20 @@ func TestProxyDial_AgentTimeout_GRPC(t *testing.T) { _, err = tunnel.DialContext(context.Background(), "tcp", blackhole) if err == nil { t.Error("Expected error when context is cancelled, did not receive error") - } else if _, reason := client.GetDialFailureReason(err); reason != client.DialFailureEndpoint { + } else if _, reason := client.GetDialFailureReason(err); reason != metricsclient.DialFailureEndpoint { t.Errorf("Unexpected error: %v", err) } - if err := metricstest.ExpectDialFailure(metrics.DialFailureErrorResponse, 1); err != nil { + if err := clientmetricstest.ExpectClientDialFailure(metrics.DialFailureEndpoint, 1); err != nil { + t.Error(err) + } + if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureErrorResponse, 1); err != nil { + t.Error(err) + } + if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureTimeout, 1); err != nil { t.Error(err) } - metrics.Metrics.Reset() // For clean shutdown. + resetAllMetrics() // For clean shutdown. select { case <-tunnel.Done(): @@ -607,10 +639,13 @@ func TestFailedDial_HTTPCONN(t *testing.T) { t.Errorf("while waiting for connection to be closed: %v", err) } - if err := metricstest.ExpectDialFailure(metrics.DialFailureErrorResponse, 1); err != nil { + if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureErrorResponse, 1); err != nil { t.Error(err) } - metrics.Metrics.Reset() // For clean shutdown. + if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureUnknown, 1); err != nil { + t.Error(err) + } + resetAllMetrics() // For clean shutdown. } func localAddr(addr net.Addr) string { @@ -770,7 +805,7 @@ func (a *unresponsiveAgent) Close() { // waitForConnectedServerCount waits for the agent ClientSet to have the expected number of health // server connections (HealthyClientsCount). -func waitForConnectedServerCount(t *testing.T, expectedServerCount int, clientset *agent.ClientSet) { +func waitForConnectedServerCount(t testing.TB, expectedServerCount int, clientset *agent.ClientSet) { t.Helper() err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { hc := clientset.HealthyClientsCount() @@ -788,7 +823,7 @@ func waitForConnectedServerCount(t *testing.T, expectedServerCount int, clientse // waitForConnectedAgentCount waits for the proxy server to have the expected number of registered // agents (backends). This assumes the ProxyServer is using a single ProxyStrategy. -func waitForConnectedAgentCount(t *testing.T, expectedAgentCount int, proxy *server.ProxyServer) { +func waitForConnectedAgentCount(t testing.TB, expectedAgentCount int, proxy *server.ProxyServer) { t.Helper() err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { count := proxy.BackendManagers[0].NumBackends() @@ -804,18 +839,40 @@ func waitForConnectedAgentCount(t *testing.T, expectedAgentCount int, proxy *ser } } -func assertNoDialFailures(t *testing.T) { +func assertNoClientDialFailures(t testing.TB) { + t.Helper() + if err := clientmetricstest.ExpectClientDialFailures(nil); err != nil { + t.Errorf("Unexpected %s metric: %v", "dial_failure_total", err) + } +} + +func assertNoServerDialFailures(t testing.TB) { t.Helper() - if err := metricstest.ExpectDialFailures(nil); err != nil { - t.Errorf("Unexpected %s metric: %v", metrics.DialFailuresMetric, err) + if err := metricstest.ExpectServerDialFailures(nil); err != nil { + t.Errorf("Unexpected %s metric: %v", metricsserver.DialFailuresMetric, err) } } -func expectCleanShutdown(t *testing.T) { - metrics.Metrics.Reset() +func assertNoAgentDialFailures(t testing.TB) { + t.Helper() + if err := metricstest.ExpectAgentDialFailures(nil); err != nil { + t.Errorf("Unexpected %s metric: %v", "endpoint_dial_failure_total", err) + } +} + +func resetAllMetrics() { + metricsclient.Metrics.Reset() + metricsserver.Metrics.Reset() + metricsagent.Metrics.Reset() +} + +func expectCleanShutdown(t testing.TB) { + resetAllMetrics() currentGoRoutines := goleak.IgnoreCurrent() t.Cleanup(func() { goleak.VerifyNone(t, currentGoRoutines) - assertNoDialFailures(t) + assertNoClientDialFailures(t) + assertNoServerDialFailures(t) + assertNoAgentDialFailures(t) }) } diff --git a/tests/readiness_test.go b/tests/readiness_test.go index 81a9cb966..ceb5521ff 100644 --- a/tests/readiness_test.go +++ b/tests/readiness_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/reconnect_test.go b/tests/reconnect_test.go index 82edd2374..f24767f64 100644 --- a/tests/reconnect_test.go +++ b/tests/reconnect_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/tcp_server_test.go b/tests/tcp_server_test.go index d0361a1fa..e8a617b3b 100644 --- a/tests/tcp_server_test.go +++ b/tests/tcp_server_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import (