From 5c4cf8a72fac494e24b058e20ac4a22fffe4ca0e Mon Sep 17 00:00:00 2001 From: Walter Fender Date: Mon, 19 Sep 2022 18:30:01 -0700 Subject: [PATCH 01/24] Adding tests for Agent options and validation. Add in the missing license files. Added requested comment on bakward compatability. --- cmd/agent/app/options/options.go | 16 +++ cmd/agent/app/options/options_test.go | 171 ++++++++++++++++++++++++ cmd/agent/app/server.go | 16 +++ cmd/server/app/options/options.go | 16 +++ cmd/server/app/options/options_test.go | 20 +++ cmd/server/app/server.go | 16 +++ pkg/agent/client_test.go | 16 +++ tests/agent_disconnect_test.go | 16 +++ tests/concurrent_client_request_test.go | 16 +++ tests/concurrent_test.go | 16 +++ tests/custom_alpn_test.go | 16 +++ tests/ha_proxy_server_test.go | 16 +++ tests/main_test.go | 16 +++ tests/proxy_test.go | 16 +++ tests/readiness_test.go | 16 +++ tests/reconnect_test.go | 16 +++ tests/tcp_server_test.go | 16 +++ 17 files changed, 431 insertions(+) create mode 100644 cmd/agent/app/options/options_test.go diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index a0e58f5dc..0ad709aaa 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package options import ( diff --git a/cmd/agent/app/options/options_test.go b/cmd/agent/app/options/options_test.go new file mode 100644 index 000000000..f4bbeed17 --- /dev/null +++ b/cmd/agent/app/options/options_test.go @@ -0,0 +1,171 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "testing" + "reflect" + "time" + "github.com/stretchr/testify/assert" + "fmt" +) + +/* + * TestDefaultServerOptions is intended to ensure we do not make a backward incompatible + * change to the default flag values for the ANP agent. + */ +func TestDefaultServerOptions(t *testing.T) { + defaultAgentOptions := NewGrpcProxyAgentOptions() + assertDefaultValue(t, "AgentCert", defaultAgentOptions.AgentCert, "") + assertDefaultValue(t, "AgentKey", defaultAgentOptions.AgentKey, "") + assertDefaultValue(t, "CaCert", defaultAgentOptions.CaCert, "") + assertDefaultValue(t, "ProxyServerHost", defaultAgentOptions.ProxyServerHost, "127.0.0.1") + assertDefaultValue(t, "ProxyServerPort", defaultAgentOptions.ProxyServerPort, 8091) + assertDefaultValue(t, "HealthServerHost", defaultAgentOptions.HealthServerHost, "") + assertDefaultValue(t, "HealthServerPort", defaultAgentOptions.HealthServerPort, 8093) + assertDefaultValue(t, "AdminServerPort", defaultAgentOptions.AdminServerPort, 8094) + assertDefaultValue(t, "EnableProfiling", defaultAgentOptions.EnableProfiling, false) + assertDefaultValue(t, "EnableContentionProfiling", defaultAgentOptions.EnableContentionProfiling, false) + assertDefaultValue(t, "AgentIdentifiers", defaultAgentOptions.AgentIdentifiers, "") + assertDefaultValue(t, "SyncInterval", defaultAgentOptions.SyncInterval, 1*time.Second) + assertDefaultValue(t, "ProbeInterval", defaultAgentOptions.ProbeInterval, 1*time.Second) + assertDefaultValue(t, "SyncIntervalCap", defaultAgentOptions.SyncIntervalCap, 10*time.Second) + assertDefaultValue(t, "KeepaliveTime", defaultAgentOptions.KeepaliveTime, 1*time.Hour) + assertDefaultValue(t, "ServiceAccountTokenPath", defaultAgentOptions.ServiceAccountTokenPath, "") + assertDefaultValue(t, "WarnOnChannelLimit", defaultAgentOptions.WarnOnChannelLimit, false) + assertDefaultValue(t, "SyncForever", defaultAgentOptions.SyncForever, false) +} + +func assertDefaultValue(t *testing.T, fieldName string, actual, expected interface{}) { + t.Helper() + assert.IsType(t, expected, actual, "For field %s, got the wrong type.", fieldName) + assert.Equal(t, expected, actual, "For field %s, got the wrong value.", fieldName) +} + +func TestValidate(t *testing.T) { + for desc, tc := range map[string]struct { + fieldMap map[string]interface{} + expected error + }{ + "default": { + fieldMap: map[string]interface{}{}, + expected: nil, + }, + "ZeroProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 0}, + expected: fmt.Errorf("proxy server port 0 must be greater than 0"), + }, + "NegativeProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": -1}, + expected: fmt.Errorf("proxy server port -1 must be greater than 0"), + }, + "ReservedProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 1023}, + expected: nil, //TODO: fmt.Errorf("please do not try to use reserved port 1023 for the proxy server port"), + }, + "StartValidProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 1024}, + expected: nil, + }, + "EndValidProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 49151}, + expected: nil, + }, + "StartEphemeralProxyServerPort": { + fieldMap: map[string]interface{}{"ProxyServerPort": 49152}, + expected: nil, //TODO: fmt.Errorf("please do not try to use ephemeral port 49152 for the proxy server port"), + }, + "ZeroHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 0}, + expected: fmt.Errorf("health server port 0 must be greater than 0"), + }, + "NegativeHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": -1}, + expected: fmt.Errorf("health server port -1 must be greater than 0"), + }, + "ReservedHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 1023}, + expected: nil, //TODO: fmt.Errorf("please do not try to use reserved port 1023 for the health server port"), + }, + "StartValidHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 1024}, + expected: nil, + }, + "EndValidHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 49151}, + expected: nil, + }, + "StartEphemeralHealthServerPort": { + fieldMap: map[string]interface{}{"HealthServerPort": 49152}, + expected: nil, //TODO: fmt.Errorf("please do not try to use ephemeral port 49152 for the health server port"), + }, + "ZeroAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 0}, + expected: fmt.Errorf("admin server port 0 must be greater than 0"), + }, + "NegativeAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": -1}, + expected: fmt.Errorf("admin server port -1 must be greater than 0"), + }, + "ReservedAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 1023}, + expected: nil, //TODO: fmt.Errorf("please do not try to use reserved port 1023 for the health port"), + }, + "StartValidAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 1024}, + expected: nil, + }, + "EndValidAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 49151}, + expected: nil, + }, + "StartEphemeralAdminServerPort": { + fieldMap: map[string]interface{}{"AdminServerPort": 49152}, + expected: nil, //TODO: fmt.Errorf("please do not try to use ephemeral port 49152 for the health port"), + }, + "ContentionProfilingRequiresProfiling": { + fieldMap: map[string]interface{}{ + "EnableContentionProfiling": true, + "EnableProfiling": false, + }, + expected: fmt.Errorf("if --enable-contention-profiling is set, --enable-profiling must also be set"), + }, + } { + t.Run(desc, func(t *testing.T) { + testAgentOptions := NewGrpcProxyAgentOptions() + for field, value := range tc.fieldMap { + rv := reflect.ValueOf(testAgentOptions) + rv = rv.Elem() + fv := rv.FieldByName(field) + switch reflect.TypeOf(value).Kind() { + case reflect.String: + svalue := value.(string) + fv.SetString(svalue) + case reflect.Int: + ivalue := value.(int) + fv.SetInt(int64(ivalue)) + case reflect.Bool: + bvalue := value.(bool) + fv.SetBool(bvalue) + } + } + actual := testAgentOptions.Validate() + assert.IsType(t, tc.expected, actual, "Validation for case %s, got the wrong type.", desc) + assert.Equal(t, tc.expected, actual, "Validation for case %s, got the wrong value.", desc) + }) + } +} diff --git a/cmd/agent/app/server.go b/cmd/agent/app/server.go index b2ee0f2af..969413985 100644 --- a/cmd/agent/app/server.go +++ b/cmd/agent/app/server.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package app import ( diff --git a/cmd/server/app/options/options.go b/cmd/server/app/options/options.go index a4269119a..94bc3ce1d 100644 --- a/cmd/server/app/options/options.go +++ b/cmd/server/app/options/options.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package options import ( diff --git a/cmd/server/app/options/options_test.go b/cmd/server/app/options/options_test.go index ad242e4ed..aa99e36ed 100644 --- a/cmd/server/app/options/options_test.go +++ b/cmd/server/app/options/options_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package options import ( @@ -8,6 +24,10 @@ import ( "github.com/stretchr/testify/assert" ) +/* + * TestDefaultServerOptions is intended to ensure we do not make a backward incompatible + * change to the default flag values for the ANP server. + */ func TestDefaultServerOptions(t *testing.T) { defaultServerOptions := NewProxyRunOptions() assertDefaultValue(t, "ServerCert", defaultServerOptions.ServerCert, "") diff --git a/cmd/server/app/server.go b/cmd/server/app/server.go index 59829803f..f7df59d10 100644 --- a/cmd/server/app/server.go +++ b/cmd/server/app/server.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package app import ( diff --git a/pkg/agent/client_test.go b/pkg/agent/client_test.go index e43e71c20..bc4b7b5e6 100644 --- a/pkg/agent/client_test.go +++ b/pkg/agent/client_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package agent import ( diff --git a/tests/agent_disconnect_test.go b/tests/agent_disconnect_test.go index 1453359a7..09bec5ff3 100644 --- a/tests/agent_disconnect_test.go +++ b/tests/agent_disconnect_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/concurrent_client_request_test.go b/tests/concurrent_client_request_test.go index bd1fafdd8..5f2f89714 100644 --- a/tests/concurrent_client_request_test.go +++ b/tests/concurrent_client_request_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/concurrent_test.go b/tests/concurrent_test.go index 8efb0fdde..d6d906c39 100644 --- a/tests/concurrent_test.go +++ b/tests/concurrent_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/custom_alpn_test.go b/tests/custom_alpn_test.go index 0ce40e3de..9f96e475e 100644 --- a/tests/custom_alpn_test.go +++ b/tests/custom_alpn_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/ha_proxy_server_test.go b/tests/ha_proxy_server_test.go index 2e703e0af..bb5ea0f91 100644 --- a/tests/ha_proxy_server_test.go +++ b/tests/ha_proxy_server_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/main_test.go b/tests/main_test.go index a9d652dd2..1df546080 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/proxy_test.go b/tests/proxy_test.go index 5fbadf5b1..dca43e695 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/readiness_test.go b/tests/readiness_test.go index 81a9cb966..ceb5521ff 100644 --- a/tests/readiness_test.go +++ b/tests/readiness_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/reconnect_test.go b/tests/reconnect_test.go index 82edd2374..f24767f64 100644 --- a/tests/reconnect_test.go +++ b/tests/reconnect_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( diff --git a/tests/tcp_server_test.go b/tests/tcp_server_test.go index d0361a1fa..e8a617b3b 100644 --- a/tests/tcp_server_test.go +++ b/tests/tcp_server_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package tests import ( From c11c5bd1d5de9b1dc64f2658ca3373ba7e6f252b Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 14 Nov 2022 16:38:38 -0500 Subject: [PATCH 02/24] update owners Signed-off-by: Andrew Sy Kim --- OWNERS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/OWNERS b/OWNERS index d97517295..50ea85c9a 100644 --- a/OWNERS +++ b/OWNERS @@ -1,15 +1,15 @@ # See the OWNERS docs at https://go.k8s.io/owners reviewers: - - caesarxuchao - dberkov - jefftree - - jkh52 + - tallclair approvers: - - cheftako - - mcrute - anfernee + - cheftako + - jkh52 +emeritus_approvers: - andrewsykim - caesarxuchao -emeritus_approvers: + - mcrute - Sh4d1 From a2dc6cf390d6de3ce7618296ec9ce41545050ba8 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Wed, 16 Nov 2022 17:29:51 -0800 Subject: [PATCH 03/24] go mod tidy --- konnectivity-client/go.mod | 3 +-- konnectivity-client/go.sum | 37 +++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/konnectivity-client/go.mod b/konnectivity-client/go.mod index e1b08efb9..65fa00ae8 100644 --- a/konnectivity-client/go.mod +++ b/konnectivity-client/go.mod @@ -13,11 +13,10 @@ require ( require ( github.com/go-logr/logr v0.1.0 // indirect - golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect golang.org/x/text v0.3.3 // indirect - golang.org/x/tools v0.1.5 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect google.golang.org/protobuf v1.26.0-rc.1 // indirect ) diff --git a/konnectivity-client/go.sum b/konnectivity-client/go.sum index ecbd23264..2d9f4b7bb 100644 --- a/konnectivity-client/go.sum +++ b/konnectivity-client/go.sum @@ -2,8 +2,9 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= @@ -34,56 +35,59 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -107,8 +111,9 @@ google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= From 456c714c333e130b6ffa40621473c3754761e69e Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Wed, 16 Nov 2022 17:31:19 -0800 Subject: [PATCH 04/24] Run konnectivity-client tests in the client module --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3472f106c..ff2a65ccc 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,8 @@ mock_gen: .PHONY: test test: - GO111MODULE=on go test -race sigs.k8s.io/apiserver-network-proxy/... + go test -race ./... + cd konnectivity-client && go test -race ./... ## -------------------------------------- ## Binaries From 73e89b0c40e7ebed2ea5817aba83333a1cb6c2f6 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Fri, 18 Nov 2022 17:41:34 -0800 Subject: [PATCH 05/24] Log the agent username on successful authentication --- pkg/server/server.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index d0902d717..e01637936 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -586,7 +586,7 @@ func getAgentIdentifiers(stream agent.AgentService_ConnectServer) (pkgagent.Iden return agentIdentifiers, nil } -func (s *ProxyServer) validateAuthToken(ctx context.Context, token string) error { +func (s *ProxyServer) validateAuthToken(ctx context.Context, token string) (username string, err error) { trReq := &authv1.TokenReview{ Spec: authv1.TokenReviewSpec{ Token: token, @@ -595,38 +595,39 @@ func (s *ProxyServer) validateAuthToken(ctx context.Context, token string) error } r, err := s.AgentAuthenticationOptions.KubernetesClient.AuthenticationV1().TokenReviews().Create(ctx, trReq, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("Failed to authenticate request. err:%v", err) + return "", fmt.Errorf("Failed to authenticate request. err:%v", err) } if r.Status.Error != "" { - return fmt.Errorf("lookup failed: %s", r.Status.Error) + return "", fmt.Errorf("lookup failed: %s", r.Status.Error) } if !r.Status.Authenticated { - return fmt.Errorf("lookup failed: service account jwt not valid") + return "", fmt.Errorf("lookup failed: service account jwt not valid") } // The username is of format: system:serviceaccount:(NAMESPACE):(SERVICEACCOUNT) - parts := strings.Split(r.Status.User.Username, ":") + username = r.Status.User.Username + parts := strings.Split(username, ":") if len(parts) != 4 { - return fmt.Errorf("lookup failed: unexpected username format") + return "", fmt.Errorf("lookup failed: unexpected username format") } // Validate the user that comes back from token review is a service account if parts[0] != "system" || parts[1] != "serviceaccount" { - return fmt.Errorf("lookup failed: username returned is not a service account") + return "", fmt.Errorf("lookup failed: username returned is not a service account") } ns := parts[2] sa := parts[3] if s.AgentAuthenticationOptions.AgentNamespace != ns { - return fmt.Errorf("lookup failed: incoming request from %q namespace. Expected %q", ns, s.AgentAuthenticationOptions.AgentNamespace) + return "", fmt.Errorf("lookup failed: incoming request from %q namespace. Expected %q", ns, s.AgentAuthenticationOptions.AgentNamespace) } if s.AgentAuthenticationOptions.AgentServiceAccount != sa { - return fmt.Errorf("lookup failed: incoming request from %q service account. Expected %q", sa, s.AgentAuthenticationOptions.AgentServiceAccount) + return "", fmt.Errorf("lookup failed: incoming request from %q service account. Expected %q", sa, s.AgentAuthenticationOptions.AgentServiceAccount) } - return nil + return username, nil } func (s *ProxyServer) authenticateAgentViaToken(ctx context.Context) error { @@ -648,11 +649,12 @@ func (s *ProxyServer) authenticateAgentViaToken(ctx context.Context) error { return fmt.Errorf("received token does not have %q prefix", header.AuthenticationTokenContextSchemePrefix) } - if err := s.validateAuthToken(ctx, strings.TrimPrefix(authContext[0], header.AuthenticationTokenContextSchemePrefix)); err != nil { + username, err := s.validateAuthToken(ctx, strings.TrimPrefix(authContext[0], header.AuthenticationTokenContextSchemePrefix)) + if err != nil { return fmt.Errorf("Failed to validate authentication token, err:%v", err) } - klog.V(2).Infoln("Client successfully authenticated via token") + klog.V(2).InfoS("Agent successfully authenticated via token", "username", username) return nil } From c3e6ddcca2194e143cf3f146940a4099a21f9145 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Fri, 18 Nov 2022 17:59:17 -0800 Subject: [PATCH 06/24] Adjust server log verbosity --- pkg/server/backend_manager.go | 6 +++--- pkg/server/server.go | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/server/backend_manager.go b/pkg/server/backend_manager.go index 6fd92026b..ab1c04c23 100644 --- a/pkg/server/backend_manager.go +++ b/pkg/server/backend_manager.go @@ -188,7 +188,7 @@ func (s *DefaultBackendStorage) AddBackend(identifier string, idType pkgagent.Id klog.V(4).InfoS("fail to add backend", "backend", identifier, "error", &ErrWrongIDType{idType, s.idTypes}) return nil } - klog.V(2).InfoS("Register backend for agent", "connection", conn, "agentID", identifier) + klog.V(5).InfoS("Register backend for agent", "connection", conn, "agentID", identifier) s.mu.Lock() defer s.mu.Unlock() _, ok := s.backends[identifier] @@ -218,7 +218,7 @@ func (s *DefaultBackendStorage) RemoveBackend(identifier string, idType pkgagent klog.ErrorS(&ErrWrongIDType{idType, s.idTypes}, "fail to remove backend") return } - klog.V(2).InfoS("Remove connection for agent", "connection", conn, "identifier", identifier) + klog.V(5).InfoS("Remove connection for agent", "connection", conn, "identifier", identifier) s.mu.Lock() defer s.mu.Unlock() backends, ok := s.backends[identifier] @@ -299,7 +299,7 @@ func (s *DefaultBackendStorage) GetRandomBackend() (Backend, error) { return nil, &ErrNotFound{} } agentID := s.agentIDs[s.random.Intn(len(s.agentIDs))] - klog.V(4).InfoS("Pick agent as backend", "agentID", agentID) + klog.V(5).InfoS("Pick agent as backend", "agentID", agentID) // always return the first connection to an agent, because the agent // will close later connections if there are multiple. return s.backends[agentID][0], nil diff --git a/pkg/server/server.go b/pkg/server/server.go index e01637936..9b03971f9 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -271,7 +271,6 @@ func (s *ProxyServer) removeBackend(agentID string, conn agent.AgentService_Conn } func (s *ProxyServer) addFrontend(agentID string, connID int64, p *ProxyClientConnection) { - klog.V(2).InfoS("Register frontend for agent", "frontend", p, "agentID", agentID, "connectionID", connID) s.fmu.Lock() defer s.fmu.Unlock() if _, ok := s.frontends[agentID]; !ok { @@ -292,12 +291,10 @@ func (s *ProxyServer) removeFrontend(agentID string, connID int64) { klog.V(2).InfoS("Cannot find connection for agent in the frontends", "connectionID", connID, "agentID", agentID) return } - klog.V(2).InfoS("Remove frontend for agent", "frontend", conns[connID], "agentID", agentID, "connectionID", connID) delete(s.frontends[agentID], connID) if len(s.frontends[agentID]) == 0 { delete(s.frontends, agentID) } - return } func (s *ProxyServer) getFrontend(agentID string, connID int64) (*ProxyClientConnection, error) { @@ -369,7 +366,7 @@ func (s *ProxyServer) Proxy(stream client.ProxyService_ProxyServer) error { return fmt.Errorf("failed to get context") } userAgent := md.Get(header.UserAgent) - klog.V(2).InfoS("Proxy request from client", "userAgent", userAgent, "serverID", s.serverID) + klog.V(5).InfoS("Proxy request from client", "userAgent", userAgent, "serverID", s.serverID) recvCh := make(chan *client.Packet, xfrChannelSize) stopCh := make(chan error) @@ -381,7 +378,6 @@ func (s *ProxyServer) Proxy(stream client.ProxyService_ProxyServer) error { go runpprof.Do(context.Background(), labels, func(context.Context) { s.serveRecvFrontend(stream, recvCh) }) defer func() { - klog.V(2).InfoS("Receive channel from frontend is stopping", "userAgent", userAgent) close(recvCh) }() @@ -395,6 +391,7 @@ func (s *ProxyServer) readFrontendToChannel(stream client.ProxyService_ProxyServ for { in, err := stream.Recv() if err == io.EOF { + // TODO: Log an error if the frontend connection stops without first closing any open connections. klog.V(2).InfoS("Receive stream from frontend closed", "userAgent", userAgent) close(stopCh) return @@ -531,6 +528,7 @@ func (s *ProxyServer) serveRecvFrontend(stream client.ProxyService_ProxyServer, } } + // TODO: Log an error if the frontend stream is closed with an established tunnel still open. klog.V(5).InfoS("Close streaming", "connectionID", firstConnID) if backend == nil { @@ -654,7 +652,7 @@ func (s *ProxyServer) authenticateAgentViaToken(ctx context.Context) error { return fmt.Errorf("Failed to validate authentication token, err:%v", err) } - klog.V(2).InfoS("Agent successfully authenticated via token", "username", username) + klog.V(5).InfoS("Agent successfully authenticated via token", "username", username) return nil } @@ -668,7 +666,7 @@ func (s *ProxyServer) Connect(stream agent.AgentService_ConnectServer) error { return err } - klog.V(2).InfoS("Connect request from agent", "agentID", agentID, "serverID", s.serverID) + klog.V(5).InfoS("Connect request from agent", "agentID", agentID, "serverID", s.serverID) labels := runpprof.Labels( "serverCount", strconv.Itoa(s.serverCount), "agentID", agentID, @@ -689,6 +687,7 @@ func (s *ProxyServer) Connect(stream agent.AgentService_ConnectServer) error { return err } + klog.V(2).InfoS("Agent connected", "agentID", agentID, "serverID", s.serverID) backend := s.addBackend(agentID, stream) defer s.removeBackend(agentID, stream) @@ -697,7 +696,6 @@ func (s *ProxyServer) Connect(stream agent.AgentService_ConnectServer) error { go runpprof.Do(context.Background(), labels, func(context.Context) { s.serveRecvBackend(backend, stream, agentID, recvCh) }) defer func() { - klog.V(2).InfoS("Receive channel from agent is stopping", "agentID", agentID) close(recvCh) }() From 0de2942a4b4d708c829ee9d1b30d7d511f81e311 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Thu, 17 Nov 2022 10:45:20 -0800 Subject: [PATCH 07/24] Improve dial response logging --- konnectivity-client/pkg/client/client.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/konnectivity-client/pkg/client/client.go b/konnectivity-client/pkg/client/client.go index c55070ae6..a2d12d01f 100644 --- a/konnectivity-client/pkg/client/client.go +++ b/konnectivity-client/pkg/client/client.go @@ -222,7 +222,11 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { // 2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context // // In either scenario, we should return here and close the tunnel as it is no longer needed. - klog.V(1).InfoS("DialResp not recognized; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random) + kvs := []interface{}{"dialID", resp.Random, "connectID", resp.ConnectID} + if resp.Error != "" { + kvs = append(kvs, "error", resp.Error) + } + klog.V(1).InfoS("DialResp not recognized; dropped", kvs...) return } From e6cff9ff71bb62bdd152841b9d5b981542373084 Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Tue, 22 Nov 2022 17:16:34 -0800 Subject: [PATCH 08/24] Agent: log dialID in DIAL_REQ handling. --- pkg/agent/client.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/agent/client.go b/pkg/agent/client.go index ada7f501e..87e0b7ccd 100644 --- a/pkg/agent/client.go +++ b/pkg/agent/client.go @@ -383,13 +383,12 @@ func (a *Client) Serve() { switch pkt.Type { case client.PacketType_DIAL_REQ: - klog.V(4).InfoS("received DIAL_REQ", "serverID", a.serverID, "agentID", a.agentID) + dialReq := pkt.GetDialRequest() + klog.V(3).InfoS("Received DIAL_REQ", "serverID", a.serverID, "agentID", a.agentID, "dialID", dialReq.Random, "dialAddress", dialReq.Address) dialResp := &client.Packet{ Type: client.PacketType_DIAL_RSP, Payload: &client.Packet_DialResponse{DialResponse: &client.DialResponse{}}, } - - dialReq := pkt.GetDialRequest() dialResp.GetDialResponse().Random = dialReq.Random connID := atomic.AddInt64(&a.nextConnID, 1) @@ -404,7 +403,7 @@ func (a *Client) Serve() { // block on purpose <-dialDone if connCtx.conn != nil { - klog.V(4).InfoS("close connection", "connectionID", connID) + klog.V(4).InfoS("close connection", "dialID", dialReq.Random, "connectionID", connID, "dialAddress", dialReq.Address) var closePkt *client.Packet if connID == 0 { closePkt = &client.Packet{ @@ -444,15 +443,16 @@ func (a *Client) Serve() { start := time.Now() conn, err := net.DialTimeout(dialReq.Protocol, dialReq.Address, dialTimeout) if err != nil { - klog.ErrorS(err, "error dialing backend", "dialID", dialReq.Random) + klog.ErrorS(err, "error dialing backend", "dialID", dialReq.Random, "connectionID", connID, "dialAddress", dialReq.Address) dialResp.GetDialResponse().Error = err.Error() if err := a.Send(dialResp); err != nil { - klog.ErrorS(err, "could not send dialResp") + klog.ErrorS(err, "could not send dialResp", "dialID", dialReq.Random, "connectionID", connID) } // Cannot invoke clean up as we have no conn yet. return } metrics.Metrics.ObserveDialLatency(time.Since(start)) + klog.V(3).InfoS("Endpoint connection established", "dialID", dialReq.Random, "connectionID", connID, "dialAddress", dialReq.Address) connCtx.conn = conn a.connManager.Add(connID, connCtx) dialResp.GetDialResponse().ConnectID = connID @@ -465,7 +465,7 @@ func (a *Client) Serve() { "dialAddress", dialReq.Address, ) if err := a.Send(dialResp); err != nil { - klog.ErrorS(err, "could not send dialResp") + klog.ErrorS(err, "could not send dialResp", "dialID", dialReq.Random) // clean-up is normally called from remoteToProxy which we will never invoke. // So we are invoking it here to force the clean-up to occur. // However, cleanup will block until dialDone is closed. @@ -473,7 +473,6 @@ func (a *Client) Serve() { go runpprof.Do(context.Background(), labels, func(context.Context) { connCtx.cleanup() }) return } - klog.V(3).InfoS("Proxying new connection", "connectionID", connID) go runpprof.Do(context.Background(), labels, func(context.Context) { a.remoteToProxy(connID, connCtx) }) go runpprof.Do(context.Background(), labels, func(context.Context) { a.proxyToRemote(connID, connCtx) }) }) @@ -521,7 +520,7 @@ func (a *Client) remoteToProxy(connID int64, ctx *connContext) { if panicInfo := recover(); panicInfo != nil { klog.V(2).InfoS("Exiting remoteToProxy with recovery", "panicInfo", panicInfo, "connectionID", connID) } else { - klog.V(3).InfoS("Exiting remoteToProxy", "connectionID", connID) + klog.V(4).InfoS("Exiting remoteToProxy", "connectionID", connID) } }() defer ctx.cleanup() @@ -564,7 +563,7 @@ func (a *Client) proxyToRemote(connID int64, ctx *connContext) { if panicInfo := recover(); panicInfo != nil { klog.V(2).InfoS("Exiting proxyToRemote with recovery", "panicInfo", panicInfo, "connectionID", connID) } else { - klog.V(3).InfoS("Exiting proxyToRemote", "connectionID", connID) + klog.V(4).InfoS("Exiting proxyToRemote", "connectionID", connID) } }() // Not safe to call cleanup here, as cleanup() closes the dataCh From 1d2756f92d1f0c1c223ebe756b842014305e5f12 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Tue, 29 Nov 2022 19:06:39 -0800 Subject: [PATCH 09/24] Add rudimentary data transfer benchmark tests --- tests/benchmarks_test.go | 157 +++++++++++++++++++++++++++++++++++++++ tests/proxy_test.go | 8 +- 2 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 tests/benchmarks_test.go diff --git a/tests/benchmarks_test.go b/tests/benchmarks_test.go new file mode 100644 index 000000000..3e06b406a --- /dev/null +++ b/tests/benchmarks_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/grpc" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" +) + +func BenchmarkLargeResponse_GRPC(b *testing.B) { + b.StopTimer() + + expectCleanShutdown(b) + + ctx := context.Background() + const length = 1 << 20 // 1M + const chunks = 10 + server := httptest.NewServer(newSizedServer(length, chunks)) + defer server.Close() + + stopCh := make(chan struct{}) + defer close(stopCh) + + proxy, cleanup, err := runGRPCProxyServer() + if err != nil { + b.Fatal(err) + } + defer cleanup() + + clientset := runAgent(proxy.agent, stopCh) + waitForConnectedServerCount(b, 1, clientset) + + req, err := http.NewRequest("GET", server.URL, nil) + if err != nil { + b.Fatal(err) + } + req.Close = true + + for n := 0; n < b.N; n++ { + // run test client + tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, proxy.front, grpc.WithInsecure()) + if err != nil { + b.Fatal(err) + } + + c := &http.Client{ + Transport: &http.Transport{ + DialContext: tunnel.DialContext, + }, + } + + r, err := c.Do(req) + if err != nil { + b.Fatal(err) + } + + b.StartTimer() // BEGIN CRITICAL SECTION + + size, err := io.Copy(io.Discard, r.Body) + if err != nil { + b.Fatal(err) + } + r.Body.Close() + + b.StopTimer() // END CRITICAL SECTION + + if size != length*chunks { + b.Fatalf("expect data length %d; got %d", length*chunks, size) + } + } +} + +func BenchmarkLargeRequest_GRPC(b *testing.B) { + b.StopTimer() + + expectCleanShutdown(b) + + const length = 10 << 20 // 10M + + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + size, err := io.Copy(io.Discard, req.Body) + if err != nil { + b.Fatal(err) + } + if size != length { + b.Fatalf("Expected data length %d; got %d", length, size) + } + req.Body.Close() + + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + stopCh := make(chan struct{}) + defer close(stopCh) + + proxy, cleanup, err := runGRPCProxyServer() + if err != nil { + b.Fatal(err) + } + defer cleanup() + + clientset := runAgent(proxy.agent, stopCh) + waitForConnectedServerCount(b, 1, clientset) + + bodyBytes := make([]byte, length) + body := bytes.NewReader(bodyBytes) + req, err := http.NewRequest("POST", server.URL, body) + if err != nil { + b.Fatal(err) + } + req.Close = true + for n := 0; n < b.N; n++ { + // run test client + tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, proxy.front, grpc.WithInsecure()) + if err != nil { + b.Fatal(err) + } + + c := &http.Client{ + Transport: &http.Transport{ + DialContext: tunnel.DialContext, + }, + } + body.Reset(bodyBytes) // We're reusing the request, so make sure to reset the body reader. + + b.StartTimer() // BEGIN CRITICAL SECTION + + if _, err := c.Do(req); err != nil { + b.Fatal(err) + } + + b.StopTimer() // END CRITICAL SECTION + } +} diff --git a/tests/proxy_test.go b/tests/proxy_test.go index 7e9d62e5b..9c3ea276c 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -786,7 +786,7 @@ func (a *unresponsiveAgent) Close() { // waitForConnectedServerCount waits for the agent ClientSet to have the expected number of health // server connections (HealthyClientsCount). -func waitForConnectedServerCount(t *testing.T, expectedServerCount int, clientset *agent.ClientSet) { +func waitForConnectedServerCount(t testing.TB, expectedServerCount int, clientset *agent.ClientSet) { t.Helper() err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { hc := clientset.HealthyClientsCount() @@ -804,7 +804,7 @@ func waitForConnectedServerCount(t *testing.T, expectedServerCount int, clientse // waitForConnectedAgentCount waits for the proxy server to have the expected number of registered // agents (backends). This assumes the ProxyServer is using a single ProxyStrategy. -func waitForConnectedAgentCount(t *testing.T, expectedAgentCount int, proxy *server.ProxyServer) { +func waitForConnectedAgentCount(t testing.TB, expectedAgentCount int, proxy *server.ProxyServer) { t.Helper() err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { count := proxy.BackendManagers[0].NumBackends() @@ -820,14 +820,14 @@ func waitForConnectedAgentCount(t *testing.T, expectedAgentCount int, proxy *ser } } -func assertNoDialFailures(t *testing.T) { +func assertNoDialFailures(t testing.TB) { t.Helper() if err := metricstest.ExpectDialFailures(nil); err != nil { t.Errorf("Unexpected %s metric: %v", metrics.DialFailuresMetric, err) } } -func expectCleanShutdown(t *testing.T) { +func expectCleanShutdown(t testing.TB) { metrics.Metrics.Reset() currentGoRoutines := goleak.IgnoreCurrent() t.Cleanup(func() { From 47b3dd44625368fee0a99c5718b0e55fdf921744 Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Tue, 6 Dec 2022 16:53:25 -0800 Subject: [PATCH 10/24] Update go.mod to be more conservative. See discussion at https://github.com/kubernetes-sigs/apiserver-network-proxy/pull/381 What I did: - match requirements with k/k release-1.24 branch - for k8s.io/api and friends, take latest 1.24 tag (v0.24.8) - delete indirect require section (to generate from scratch) - go mod tidy --- go.mod | 68 ++++++++++++++-------------- go.sum | 141 +++++++++++++++++++++------------------------------------ 2 files changed, 87 insertions(+), 122 deletions(-) diff --git a/go.mod b/go.mod index 0122b0a75..73f0a5a17 100644 --- a/go.mod +++ b/go.mod @@ -5,65 +5,67 @@ go 1.18 require ( github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.2 - github.com/google/uuid v1.3.0 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 - github.com/spf13/cobra v1.5.0 + github.com/google/uuid v1.1.2 + github.com/prometheus/client_golang v1.12.1 + github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.0 go.uber.org/goleak v1.2.0 - golang.org/x/net v0.0.0-20220812174116-3211cb980234 + golang.org/x/net v0.1.1-0.20221027164007-c63010009c80 google.golang.org/grpc v1.48.0 - k8s.io/api v0.24.3 - k8s.io/apimachinery v0.24.3 - k8s.io/client-go v0.24.3 - k8s.io/component-base v0.24.3 - k8s.io/klog/v2 v2.70.1 + k8s.io/api v0.24.8 + k8s.io/apimachinery v0.24.8 + k8s.io/client-go v0.24.8 + k8s.io/component-base v0.24.8 + k8s.io/klog/v2 v2.60.1 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.0 ) require ( + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.22.0 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/gnostic v0.6.9 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/imdario/mergo v0.3.13 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/imdario/mergo v0.3.5 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.7.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect - golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/term v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220812140447-cec7f5303424 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea // indirect - k8s.io/utils v0.0.0-20220812165043-ad590609e2e5 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect ) replace sigs.k8s.io/apiserver-network-proxy/konnectivity-client => ./konnectivity-client diff --git a/go.sum b/go.sum index 5d8b209b3..aafd21614 100644 --- a/go.sum +++ b/go.sum @@ -48,7 +48,9 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -65,7 +67,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -84,7 +85,6 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -93,9 +93,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -108,10 +107,10 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -121,28 +120,23 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.0 h1:1VXunYCNgapcSzFtcY+eBmrwESlYCnFJZahQRgTRoo8= -github.com/go-openapi/swag v0.22.0/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -183,9 +177,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= -github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -197,12 +190,11 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -218,9 +210,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -232,12 +223,10 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -256,7 +245,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -264,9 +252,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -286,18 +273,19 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -309,9 +297,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -320,16 +307,14 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -338,9 +323,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -357,15 +341,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -397,7 +377,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -421,9 +400,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -435,7 +413,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -476,13 +453,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= -golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.1-0.20221027164007-c63010009c80 h1:CtRWmqbiPSOXwJV1JoY7pWiTx2xzVKQ813bvU+Y/9jI= +golang.org/x/net v0.1.1-0.20221027164007-c63010009c80/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -495,10 +469,8 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7 h1:dtndE8FcEta75/4kHF3AbpuWzV6f1LjnLrM4pe2SZrw= -golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -563,16 +535,15 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -581,14 +552,14 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -642,11 +613,11 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -718,9 +689,8 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220812140447-cec7f5303424 h1:zZnTt15U44/Txe/9cN/tVbteBkPMiyXK48hPsKRmqj4= -google.golang.org/genproto v0.0.0-20220812140447-cec7f5303424/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -756,19 +726,19 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -782,7 +752,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= @@ -794,37 +763,31 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.24.3 h1:tt55QEmKd6L2k5DP6G/ZzdMQKvG5ro4H4teClqm0sTY= -k8s.io/api v0.24.3/go.mod h1:elGR/XSZrS7z7cSZPzVWaycpJuGIw57j9b95/1PdJNI= -k8s.io/apimachinery v0.24.3 h1:hrFiNSA2cBZqllakVYyH/VyEh4B581bQRmqATJSeQTg= -k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/client-go v0.24.3 h1:Nl1840+6p4JqkFWEW2LnMKU667BUxw03REfLAVhuKQY= -k8s.io/client-go v0.24.3/go.mod h1:AAovolf5Z9bY1wIg2FZ8LPQlEdKHjLI7ZD4rw920BJw= -k8s.io/component-base v0.24.3 h1:u99WjuHYCRJjS1xeLOx72DdRaghuDnuMgueiGMFy1ec= -k8s.io/component-base v0.24.3/go.mod h1:bqom2IWN9Lj+vwAkPNOv2TflsP1PeVDIwIN0lRthxYY= +k8s.io/api v0.24.8 h1:5bZ6aotI1J+BG2g0U9nBrPy1dOzBeJ/HFOHi6dX+ud4= +k8s.io/api v0.24.8/go.mod h1:W2RSRCK+eDrYEH4YeuSKrIY90TYYIcW1ojk8Mo6HVOo= +k8s.io/apimachinery v0.24.8 h1:/xehDgfpC4uN7I1GBVvx+Anwb2Jjem+GyJ9F9lSmkBs= +k8s.io/apimachinery v0.24.8/go.mod h1:WR5z9Lpw2mOAeDg20iSSrEBRQMY0p2YXVdYpUIgSr4o= +k8s.io/client-go v0.24.8 h1:drY++vKWtstFnG5Z1/4KkViBWRFaaiUugOfM4emNvyE= +k8s.io/client-go v0.24.8/go.mod h1:YrQVH7HVsT8VIsFgzLUrXHVq9RRt13hA9RoDQ8It76g= +k8s.io/component-base v0.24.8 h1:3qjRp7V+lj8b3HmZP6kF21EaKU0femkj78Gvw7cPU5M= +k8s.io/component-base v0.24.8/go.mod h1:jYVLUn7NDJs0a8CtuPSudgLEuW/PMrjeCALZZCS7K2E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea h1:3QOH5+2fGsY8e1qf+GIFpg+zw/JGNrgyZRQR7/m6uWg= -k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220812165043-ad590609e2e5 h1:XmRqFcQlCy/lKRZ39j+RVpokYNroHPqV3mcBRfnhT5o= -k8s.io/utils v0.0.0-20220812165043-ad590609e2e5/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 4a8d316aaf9ff5e27ea34800cf8b55d5528de9e9 Mon Sep 17 00:00:00 2001 From: Walter Fender Date: Tue, 6 Dec 2022 23:49:52 -0800 Subject: [PATCH 11/24] Fixed admin port listen code. Made sure we properly constructed the admin listener. Make sure we honored the bind address for the Konn Server. Added a bind address option for the Konn Agent. --- cmd/agent/app/options/options.go | 11 +++++++++-- cmd/agent/app/options/options_test.go | 1 + cmd/agent/app/server.go | 2 +- cmd/server/app/server.go | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index 273f64937..7a60eefeb 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -42,10 +42,14 @@ type GrpcProxyAgentOptions struct { ProxyServerPort int AlpnProtos []string - // Ports for the health and admin server + // Bind address for the health connections. HealthServerHost string + // Port we listen for health connections on. HealthServerPort int - AdminServerPort int + // Bind address for the admin connections. + AdminBindAddress string + // Port we listen for admin connections on. + AdminServerPort int // Enables pprof at host:adminPort/debug/pprof. EnableProfiling bool // If EnableProfiling is true, this enables the lock contention @@ -100,6 +104,7 @@ func (o *GrpcProxyAgentOptions) Flags() *pflag.FlagSet { flags.StringVar(&o.HealthServerHost, "health-server-host", o.HealthServerHost, "The host address to listen on, without port.") flags.IntVar(&o.HealthServerPort, "health-server-port", o.HealthServerPort, "The port the health server is listening on.") flags.IntVar(&o.AdminServerPort, "admin-server-port", o.AdminServerPort, "The port the admin server is listening on.") + flags.StringVar(&o.AdminBindAddress, "admin-bind-address", o.AdminBindAddress, "Bind address for admin connections. If empty, we will bind to all interfaces.") flags.BoolVar(&o.EnableProfiling, "enable-profiling", o.EnableProfiling, "enable pprof at host:admin-port/debug/pprof") flags.BoolVar(&o.EnableContentionProfiling, "enable-contention-profiling", o.EnableContentionProfiling, "enable contention profiling at host:admin-port/debug/pprof/block. \"--enable-profiling\" must also be set.") flags.StringVar(&o.AgentID, "agent-id", o.AgentID, "The unique ID of this agent. Can also be set by the 'PROXY_AGENT_ID' environment variable. Default to a generated uuid if not set.") @@ -123,6 +128,7 @@ func (o *GrpcProxyAgentOptions) Print() { klog.V(1).Infof("ALPNProtos set to %+s.\n", o.AlpnProtos) klog.V(1).Infof("HealthServerHost set to %s\n", o.HealthServerHost) klog.V(1).Infof("HealthServerPort set to %d.\n", o.HealthServerPort) + klog.V(1).Infof("Admin bind address set to %q.\n", o.AdminBindAddress) klog.V(1).Infof("AdminServerPort set to %d.\n", o.AdminServerPort) klog.V(1).Infof("EnableProfiling set to %v.\n", o.EnableProfiling) klog.V(1).Infof("EnableContentionProfiling set to %v.\n", o.EnableContentionProfiling) @@ -213,6 +219,7 @@ func NewGrpcProxyAgentOptions() *GrpcProxyAgentOptions { ProxyServerPort: 8091, HealthServerHost: "", HealthServerPort: 8093, + AdminBindAddress: "127.0.0.1", AdminServerPort: 8094, EnableProfiling: false, EnableContentionProfiling: false, diff --git a/cmd/agent/app/options/options_test.go b/cmd/agent/app/options/options_test.go index f4bbeed17..bb830a06b 100644 --- a/cmd/agent/app/options/options_test.go +++ b/cmd/agent/app/options/options_test.go @@ -37,6 +37,7 @@ func TestDefaultServerOptions(t *testing.T) { assertDefaultValue(t, "ProxyServerPort", defaultAgentOptions.ProxyServerPort, 8091) assertDefaultValue(t, "HealthServerHost", defaultAgentOptions.HealthServerHost, "") assertDefaultValue(t, "HealthServerPort", defaultAgentOptions.HealthServerPort, 8093) + assertDefaultValue(t, "AdminBindAddress", defaultAgentOptions.AdminBindAddress, "127.0.0.1") assertDefaultValue(t, "AdminServerPort", defaultAgentOptions.AdminServerPort, 8094) assertDefaultValue(t, "EnableProfiling", defaultAgentOptions.EnableProfiling, false) assertDefaultValue(t, "EnableContentionProfiling", defaultAgentOptions.EnableContentionProfiling, false) diff --git a/cmd/agent/app/server.go b/cmd/agent/app/server.go index 969413985..1fa9411e9 100644 --- a/cmd/agent/app/server.go +++ b/cmd/agent/app/server.go @@ -158,7 +158,7 @@ func (a *Agent) runAdminServer(o *options.GrpcProxyAgentOptions) error { } adminServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", o.AdminServerPort), + Addr: net.JoinHostPort(o.AdminBindAddress, strconv.Itoa(o.AdminServerPort)), Handler: muxHandler, MaxHeaderBytes: 1 << 20, ReadHeaderTimeout: ReadHeaderTimeout, diff --git a/cmd/server/app/server.go b/cmd/server/app/server.go index f7df59d10..71075124d 100644 --- a/cmd/server/app/server.go +++ b/cmd/server/app/server.go @@ -396,7 +396,7 @@ func (p *Proxy) runAdminServer(o *options.ProxyRunOptions, server *server.ProxyS } } adminServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", o.AdminPort), + Addr: net.JoinHostPort(o.AdminBindAddress, strconv.Itoa(o.AdminPort)), Handler: muxHandler, MaxHeaderBytes: 1 << 20, } From 869e056ba6aa811fb7c529d28365970c682da8f6 Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Sun, 13 Nov 2022 23:39:35 -0800 Subject: [PATCH 12/24] Add several new metrics, including new KAS client package. --- konnectivity-client/go.mod | 6 + konnectivity-client/go.sum | 67 ++++++--- konnectivity-client/pkg/client/client.go | 28 +++- konnectivity-client/pkg/client/client_test.go | 9 ++ konnectivity-client/pkg/client/conn.go | 9 ++ .../pkg/client/metrics/metrics.go | 86 +++++++++++ .../pkg/common/metrics/metrics.go | 78 ++++++++++ pkg/agent/client.go | 20 ++- pkg/agent/clientset.go | 3 + pkg/agent/metrics/metrics.go | 134 +++++++++++++++--- pkg/server/backend_manager.go | 10 +- pkg/server/metrics/metrics.go | 96 ++++++++----- pkg/server/server.go | 23 ++- pkg/testing/metrics/metrics.go | 38 +++-- tests/proxy_test.go | 50 +++++-- 15 files changed, 549 insertions(+), 108 deletions(-) create mode 100644 konnectivity-client/pkg/client/metrics/metrics.go create mode 100644 konnectivity-client/pkg/common/metrics/metrics.go diff --git a/konnectivity-client/go.mod b/konnectivity-client/go.mod index 65fa00ae8..a29c22342 100644 --- a/konnectivity-client/go.mod +++ b/konnectivity-client/go.mod @@ -6,13 +6,19 @@ go 1.17 // k/k minor version, to prevent client backport issues. require ( github.com/golang/protobuf v1.4.3 + github.com/prometheus/client_golang v1.0.0 go.uber.org/goleak v1.2.0 google.golang.org/grpc v1.27.1 k8s.io/klog/v2 v2.0.0 ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/go-logr/logr v0.1.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect + github.com/prometheus/common v0.4.1 // indirect + github.com/prometheus/procfs v0.1.3 // indirect golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect golang.org/x/text v0.3.3 // indirect diff --git a/konnectivity-client/go.sum b/konnectivity-client/go.sum index 2d9f4b7bb..4236926e1 100644 --- a/konnectivity-client/go.sum +++ b/konnectivity-client/go.sum @@ -1,5 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -7,12 +13,17 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -28,45 +39,64 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= @@ -80,11 +110,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -109,11 +135,10 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= diff --git a/konnectivity-client/pkg/client/client.go b/konnectivity-client/pkg/client/client.go index a2d12d01f..ec6489338 100644 --- a/konnectivity-client/pkg/client/client.go +++ b/konnectivity-client/pkg/client/client.go @@ -29,6 +29,9 @@ import ( "google.golang.org/grpc" "k8s.io/klog/v2" + + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) @@ -139,6 +142,11 @@ type clientConn interface { var _ clientConn = &grpc.ClientConn{} +var ( + // Expose metrics for client to register. + Metrics = metrics.Metrics +) + // CreateSingleUseGrpcTunnel creates a Tunnel to dial to a remote server through a // gRPC based proxy service. // Currently, a single tunnel supports a single connection, and the tunnel is closed when the connection is terminated @@ -201,14 +209,22 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { for { pkt, err := t.stream.Recv() - if err == io.EOF || t.isClosing() { + if err == io.EOF { return } + const segment = commonmetrics.SegmentToClient + isClosing := t.isClosing() if err != nil || pkt == nil { - klog.ErrorS(err, "stream read failure") + if !isClosing { + klog.ErrorS(err, "stream read failure") + } + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) + return + } + metrics.Metrics.ObservePacket(segment, pkt.Type) + if isClosing { return } - klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type) switch pkt.Type { @@ -354,8 +370,11 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s } klog.V(5).InfoS("[tracing] send packet", "type", req.Type) + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, req.Type) err := t.stream.Send(req) if err != nil { + metrics.Metrics.ObserveStreamError(segment, err, req.Type) return nil, err } @@ -406,7 +425,10 @@ func (t *grpcTunnel) closeDial(dialID int64) { }, }, } + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, req.Type) if err := t.stream.Send(req); err != nil { + metrics.Metrics.ObserveStreamError(segment, err, req.Type) klog.V(5).InfoS("Failed to send DIAL_CLS", "err", err, "dialID", dialID) } t.closeTunnel() diff --git a/konnectivity-client/pkg/client/client_test.go b/konnectivity-client/pkg/client/client_test.go index a4ef71176..e5de029c7 100644 --- a/konnectivity-client/pkg/client/client_test.go +++ b/konnectivity-client/pkg/client/client_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/goleak" "google.golang.org/grpc" "k8s.io/klog/v2" @@ -448,6 +449,14 @@ func TestDial_Closed(t *testing.T) { }() } +func TestRegisterMetrics(t *testing.T) { + Metrics.RegisterMetrics(prometheus.DefaultRegisterer, "namespace", "subsystem") +} + +func TestLegacyRegisterMetrics(t *testing.T) { + Metrics.LegacyRegisterMetrics(prometheus.MustRegister, "namespace", "subsystem") +} + // TODO: Move to common testing library // fakeStream implements ProxyService_ProxyClient diff --git a/konnectivity-client/pkg/client/conn.go b/konnectivity-client/pkg/client/conn.go index f76b1e37a..14384a62c 100644 --- a/konnectivity-client/pkg/client/conn.go +++ b/konnectivity-client/pkg/client/conn.go @@ -23,6 +23,9 @@ import ( "time" "k8s.io/klog/v2" + + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) @@ -62,8 +65,11 @@ func (c *conn) Write(data []byte) (n int, err error) { klog.V(5).InfoS("[tracing] send req", "type", req.Type) + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, req.Type) err = c.stream.Send(req) if err != nil { + metrics.Metrics.ObserveStreamError(segment, err, req.Type) return 0, err } return len(data), err @@ -147,7 +153,10 @@ func (c *conn) Close() error { klog.V(5).InfoS("[tracing] send req", "type", req.Type) + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, req.Type) if err := c.stream.Send(req); err != nil { + metrics.Metrics.ObserveStreamError(segment, err, req.Type) return err } diff --git a/konnectivity-client/pkg/client/metrics/metrics.go b/konnectivity-client/pkg/client/metrics/metrics.go new file mode 100644 index 000000000..afef1c772 --- /dev/null +++ b/konnectivity-client/pkg/client/metrics/metrics.go @@ -0,0 +1,86 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" +) + +const ( + Namespace = "konnectivity_network_proxy" + Subsystem = "client" +) + +var ( + // Metrics provides access to all client metrics. The client + // application is responsible for registering (via Metrics.RegisterMetrics). + Metrics = newMetrics() +) + +// ClientMetrics includes all the metrics of the konnectivity-client. +type ClientMetrics struct { + registerOnce sync.Once + streamPackets *prometheus.CounterVec + streamErrors *prometheus.CounterVec +} + +func newMetrics() *ClientMetrics { + return &ClientMetrics{ + streamPackets: commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem), + streamErrors: commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem), + } +} + +// RegisterMetrics registers all metrics with the client application. +func (c *ClientMetrics) RegisterMetrics(r prometheus.Registerer, namespace, subsystem string) { + c.registerOnce.Do(func() { + r.MustRegister(c.streamPackets) + r.MustRegister(c.streamErrors) + }) +} + +// LegacyRegisterMetrics registers all metrics via MustRegister func. +// TODO: remove this once https://github.com/kubernetes/kubernetes/pull/114293 is available. +func (c *ClientMetrics) LegacyRegisterMetrics(mustRegisterFn func(...prometheus.Collector), namespace, subsystem string) { + c.registerOnce.Do(func() { + mustRegisterFn(c.streamPackets) + mustRegisterFn(c.streamErrors) + }) +} + +// Reset resets the metrics. +func (c *ClientMetrics) Reset() { + c.streamPackets.Reset() + c.streamErrors.Reset() +} + +func (c *ClientMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { + commonmetrics.ObservePacket(c.streamPackets, segment, packetType) +} + +func (c *ClientMetrics) ObserveStreamErrorNoPacket(segment commonmetrics.Segment, err error) { + commonmetrics.ObserveStreamErrorNoPacket(c.streamErrors, segment, err) +} + +func (c *ClientMetrics) ObserveStreamError(segment commonmetrics.Segment, err error, packetType client.PacketType) { + commonmetrics.ObserveStreamError(c.streamErrors, segment, err, packetType) +} diff --git a/konnectivity-client/pkg/common/metrics/metrics.go b/konnectivity-client/pkg/common/metrics/metrics.go new file mode 100644 index 000000000..e8619f472 --- /dev/null +++ b/konnectivity-client/pkg/common/metrics/metrics.go @@ -0,0 +1,78 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package metrics provides metric definitions and helpers used +// across konnectivity client, server, and agent. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/status" + + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" +) + +// Segment identifies one of four tunnel segments (e.g. from server to agent). +type Segment string + +const ( + // SegmentFromClient indicates a packet from client to server. + SegmentFromClient Segment = "from_client" + // SegmentToClient indicates a packet from server to client. + SegmentToClient Segment = "to_client" + // SegmentFromAgent indicates a packet from agent to server. + SegmentFromAgent Segment = "from_agent" + // SegmentToAgent indicates a packet from server to agent. + SegmentToAgent Segment = "to_agent" +) + +func MakeStreamPacketsTotalMetric(namespace, subsystem string) *prometheus.CounterVec { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "stream_packets_total", + Help: "Count of packets processed, by segment and packet type (example: from_client, DIAL_REQ)", + }, + []string{"segment", "packet_type"}, + ) +} + +func MakeStreamErrorsTotalMetric(namespace, subsystem string) *prometheus.CounterVec { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "stream_errors_total", + Help: "Count of gRPC stream errors, by segment, grpc Code, packet type. (example: from_agent, Code.Unavailable, DIAL_RSP)", + }, + []string{"segment", "code", "packet_type"}, + ) +} + +func ObservePacket(m *prometheus.CounterVec, segment Segment, packetType client.PacketType) { + m.WithLabelValues(string(segment), packetType.String()).Inc() +} + +func ObserveStreamErrorNoPacket(m *prometheus.CounterVec, segment Segment, err error) { + code := status.Code(err) + m.WithLabelValues(string(segment), code.String(), "Unknown").Inc() +} + +func ObserveStreamError(m *prometheus.CounterVec, segment Segment, err error, packetType client.PacketType) { + code := status.Code(err) + m.WithLabelValues(string(segment), code.String(), packetType.String()).Inc() +} diff --git a/pkg/agent/client.go b/pkg/agent/client.go index 87e0b7ccd..51a0b378d 100644 --- a/pkg/agent/client.go +++ b/pkg/agent/client.go @@ -33,6 +33,8 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/metadata" "k8s.io/klog/v2" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" "sigs.k8s.io/apiserver-network-proxy/proto/agent" @@ -80,6 +82,7 @@ type connectionManager struct { func (cm *connectionManager) Add(connID int64, ctx *connContext) { cm.mu.Lock() defer cm.mu.Unlock() + metrics.Metrics.EndpointConnectionInc() cm.connections[connID] = ctx } @@ -93,6 +96,9 @@ func (cm *connectionManager) Get(connID int64) (*connContext, bool) { func (cm *connectionManager) Delete(connID int64) { cm.mu.Lock() defer cm.mu.Unlock() + // Delete for a connID is called from cleanFunc, which is + // protected by cleanOnce. + metrics.Metrics.EndpointConnectionDec() delete(cm.connections, connID) } @@ -272,9 +278,12 @@ func (a *Client) Send(pkt *client.Packet) error { a.sendLock.Lock() defer a.sendLock.Unlock() + const segment = commonmetrics.SegmentFromAgent + metrics.Metrics.ObservePacket(segment, pkt.Type) err := a.stream.Send(pkt) if err != nil && err != io.EOF { - metrics.Metrics.ObserveFailure(metrics.DirectionToServer) + metrics.Metrics.ObserveServerFailureDeprecated(metrics.DirectionToServer) + metrics.Metrics.ObserveStreamError(segment, err, pkt.Type) a.cs.RemoveClient(a.serverID) } return err @@ -286,7 +295,8 @@ func (a *Client) Recv() (*client.Packet, error) { pkt, err := a.stream.Recv() if err != nil && err != io.EOF { - metrics.Metrics.ObserveFailure(metrics.DirectionFromServer) + metrics.Metrics.ObserveServerFailureDeprecated(metrics.DirectionFromServer) + metrics.Metrics.ObserveStreamErrorNoPacket(commonmetrics.SegmentToAgent, err) } return pkt, err } @@ -381,6 +391,7 @@ func (a *Client) Serve() { continue } + metrics.Metrics.ObservePacket(commonmetrics.SegmentToAgent, pkt.Type) switch pkt.Type { case client.PacketType_DIAL_REQ: dialReq := pkt.GetDialRequest() @@ -443,6 +454,11 @@ func (a *Client) Serve() { start := time.Now() conn, err := net.DialTimeout(dialReq.Protocol, dialReq.Address, dialTimeout) if err != nil { + reason := metrics.DialFailureUnknown + if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + reason = metrics.DialFailureTimeout + } + metrics.Metrics.ObserveDialFailure(reason) klog.ErrorS(err, "error dialing backend", "dialID", dialReq.Random, "connectionID", connID, "dialAddress", dialReq.Address) dialResp.GetDialResponse().Error = err.Error() if err := a.Send(dialResp); err != nil { diff --git a/pkg/agent/clientset.go b/pkg/agent/clientset.go index dfb3944ff..4759e1d97 100644 --- a/pkg/agent/clientset.go +++ b/pkg/agent/clientset.go @@ -27,6 +27,7 @@ import ( "google.golang.org/grpc/connectivity" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" ) // ClientSet consists of clients connected to each instance of an HA proxy server. @@ -105,6 +106,7 @@ func (cs *ClientSet) addClientLocked(serverID string, c *Client) error { return &DuplicateServerError{ServerID: serverID} } cs.clients[serverID] = c + metrics.Metrics.SetServerConnectionsCount(len(cs.clients)) return nil } @@ -123,6 +125,7 @@ func (cs *ClientSet) RemoveClient(serverID string) { } cs.clients[serverID].Close() delete(cs.clients, serverID) + metrics.Metrics.SetServerConnectionsCount(len(cs.clients)) } type ClientSetConfig struct { diff --git a/pkg/agent/metrics/metrics.go b/pkg/agent/metrics/metrics.go index d12b29ed9..7a7ac19b8 100644 --- a/pkg/agent/metrics/metrics.go +++ b/pkg/agent/metrics/metrics.go @@ -20,13 +20,16 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) type Direction string const ( - namespace = "konnectivity_network_proxy" - subsystem = "agent" + Namespace = "konnectivity_network_proxy" + Subsystem = "agent" // DirectionToServer indicates that the agent attempts to send a packet // to the proxy server. @@ -46,49 +49,140 @@ var ( // AgentMetrics includes all the metrics of the proxy agent. type AgentMetrics struct { - latencies *prometheus.HistogramVec - failures *prometheus.CounterVec + dialLatencies *prometheus.HistogramVec + serverFailures *prometheus.CounterVec + dialFailures *prometheus.CounterVec + serverConnections *prometheus.GaugeVec + endpointConnections *prometheus.GaugeVec + streamPackets *prometheus.CounterVec + streamErrors *prometheus.CounterVec } // newAgentMetrics create a new AgentMetrics, configured with default metric names. func newAgentMetrics() *AgentMetrics { - latencies := prometheus.NewHistogramVec( + dialLatencies := prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, + Namespace: Namespace, + Subsystem: Subsystem, Name: "dial_duration_seconds", Help: "Latency of dial to the remote endpoint in seconds", Buckets: latencyBuckets, }, []string{}, ) - failures := prometheus.NewCounterVec( + serverFailures := prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, + Namespace: Namespace, + Subsystem: Subsystem, Name: "server_connection_failure_count", - Help: "Count of failures to send to or receive from the proxy server, labeled by the direction (from_server or to_server)", + Help: "Count of failures to send to or receive from the proxy server, labeled by the direction (from_server or to_server). DEPRECATED, please use stream_events_error_total", }, []string{"direction"}, ) - prometheus.MustRegister(failures) - prometheus.MustRegister(latencies) - return &AgentMetrics{failures: failures, latencies: latencies} + dialFailures := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "endpoint_dial_failure_total", + Help: "Number of failures dialing the remote endpoint, by reason (example: timeout).", + }, + []string{"reason"}, + ) + serverConnections := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "open_server_connections", + Help: "Current number of open server connections.", + }, + []string{}, + ) + endpointConnections := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "open_endpoint_connections", + Help: "Current number of open endpoint connections.", + }, + []string{}, + ) + streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem) + streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem) + prometheus.MustRegister(dialLatencies) + prometheus.MustRegister(serverFailures) + prometheus.MustRegister(dialFailures) + prometheus.MustRegister(serverConnections) + prometheus.MustRegister(endpointConnections) + prometheus.MustRegister(streamPackets) + prometheus.MustRegister(streamErrors) + return &AgentMetrics{ + dialLatencies: dialLatencies, + serverFailures: serverFailures, + dialFailures: dialFailures, + serverConnections: serverConnections, + endpointConnections: endpointConnections, + streamPackets: streamPackets, + streamErrors: streamErrors, + } + } // Reset resets the metrics. func (a *AgentMetrics) Reset() { - a.failures.Reset() - a.latencies.Reset() + a.dialLatencies.Reset() + a.serverFailures.Reset() + a.dialFailures.Reset() + a.serverConnections.Reset() + a.endpointConnections.Reset() + a.streamPackets.Reset() + a.streamErrors.Reset() } -// ObserveFailure records a failure to send to or receive from the proxy +// ObserveServerFailure records a failure to send to or receive from the proxy // server, labeled by the direction. -func (a *AgentMetrics) ObserveFailure(direction Direction) { - a.failures.WithLabelValues(string(direction)).Inc() +func (a *AgentMetrics) ObserveServerFailureDeprecated(direction Direction) { + a.serverFailures.WithLabelValues(string(direction)).Inc() } +type DialFailureReason string + +const ( + DialFailureTimeout DialFailureReason = "timeout" + DialFailureUnknown DialFailureReason = "unknown" +) + // ObserveDialLatency records the latency of dial to the remote endpoint. func (a *AgentMetrics) ObserveDialLatency(elapsed time.Duration) { - a.latencies.WithLabelValues().Observe(elapsed.Seconds()) + a.dialLatencies.WithLabelValues().Observe(elapsed.Seconds()) +} + +// ObserveDialFailure records a remote endpoint dial failure. +func (a *AgentMetrics) ObserveDialFailure(reason DialFailureReason) { + a.dialFailures.WithLabelValues(string(reason)).Inc() +} + +func (a *AgentMetrics) SetServerConnectionsCount(count int) { + a.serverConnections.WithLabelValues().Set(float64(count)) +} + +// EndpointConnectionInc increments a new endpoint connection. +func (a *AgentMetrics) EndpointConnectionInc() { + a.endpointConnections.WithLabelValues().Inc() +} + +// EndpointConnectionDec decrements a finished endpoint connection. +func (a *AgentMetrics) EndpointConnectionDec() { + a.endpointConnections.WithLabelValues().Dec() +} + +func (a *AgentMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { + commonmetrics.ObservePacket(a.streamPackets, segment, packetType) +} + +func (a *AgentMetrics) ObserveStreamErrorNoPacket(segment commonmetrics.Segment, err error) { + commonmetrics.ObserveStreamErrorNoPacket(a.streamErrors, segment, err) +} + +func (a *AgentMetrics) ObserveStreamError(segment commonmetrics.Segment, err error, packetType client.PacketType) { + commonmetrics.ObserveStreamError(a.streamErrors, segment, err, packetType) } diff --git a/pkg/server/backend_manager.go b/pkg/server/backend_manager.go index ab1c04c23..15e5c97cc 100644 --- a/pkg/server/backend_manager.go +++ b/pkg/server/backend_manager.go @@ -25,6 +25,8 @@ import ( "time" "k8s.io/klog/v2" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent" "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" @@ -85,7 +87,13 @@ type backend struct { func (b *backend) Send(p *client.Packet) error { b.mu.Lock() defer b.mu.Unlock() - return b.conn.Send(p) + const segment = commonmetrics.SegmentToAgent + metrics.Metrics.ObservePacket(segment, p.Type) + err := b.conn.Send(p) + if err != nil { + metrics.Metrics.ObserveStreamError(segment, err, p.Type) + } + return err } func (b *backend) Context() context.Context { diff --git a/pkg/server/metrics/metrics.go b/pkg/server/metrics/metrics.go index 6209071df..fde6b4ded 100644 --- a/pkg/server/metrics/metrics.go +++ b/pkg/server/metrics/metrics.go @@ -20,6 +20,9 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) const ( @@ -51,19 +54,21 @@ var ( // ServerMetrics includes all the metrics of the proxy server. type ServerMetrics struct { - latencies *prometheus.HistogramVec + endpointLatencies *prometheus.HistogramVec frontendLatencies *prometheus.HistogramVec - connections *prometheus.GaugeVec + grpcConnections *prometheus.GaugeVec httpConnections prometheus.Gauge backend *prometheus.GaugeVec pendingDials *prometheus.GaugeVec fullRecvChannels *prometheus.GaugeVec dialFailures *prometheus.CounterVec + streamPackets *prometheus.CounterVec + streamErrors *prometheus.CounterVec } // newServerMetrics create a new ServerMetrics, configured with default metric names. func newServerMetrics() *ServerMetrics { - latencies := prometheus.NewHistogramVec( + endpointLatencies := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: Namespace, Subsystem: Subsystem, @@ -83,7 +88,7 @@ func newServerMetrics() *ServerMetrics { }, []string{}, ) - connections := prometheus.NewGaugeVec( + grpcConnections := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: Subsystem, @@ -142,77 +147,84 @@ func newServerMetrics() *ServerMetrics { "reason", }, ) - - prometheus.MustRegister(latencies) + streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem) + streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem) + prometheus.MustRegister(endpointLatencies) prometheus.MustRegister(frontendLatencies) - prometheus.MustRegister(connections) + prometheus.MustRegister(grpcConnections) prometheus.MustRegister(httpConnections) prometheus.MustRegister(backend) prometheus.MustRegister(pendingDials) prometheus.MustRegister(fullRecvChannels) prometheus.MustRegister(dialFailures) + prometheus.MustRegister(streamPackets) + prometheus.MustRegister(streamErrors) return &ServerMetrics{ - latencies: latencies, + endpointLatencies: endpointLatencies, frontendLatencies: frontendLatencies, - connections: connections, + grpcConnections: grpcConnections, httpConnections: httpConnections, backend: backend, pendingDials: pendingDials, fullRecvChannels: fullRecvChannels, dialFailures: dialFailures, + streamPackets: streamPackets, + streamErrors: streamErrors, } } // Reset resets the metrics. -func (a *ServerMetrics) Reset() { - a.latencies.Reset() - a.frontendLatencies.Reset() - a.connections.Reset() - a.backend.Reset() - a.pendingDials.Reset() - a.fullRecvChannels.Reset() - a.dialFailures.Reset() +func (s *ServerMetrics) Reset() { + s.endpointLatencies.Reset() + s.frontendLatencies.Reset() + s.grpcConnections.Reset() + s.backend.Reset() + s.pendingDials.Reset() + s.fullRecvChannels.Reset() + s.dialFailures.Reset() + s.streamPackets.Reset() + s.streamErrors.Reset() } // ObserveDialLatency records the latency of dial to the remote endpoint. -func (a *ServerMetrics) ObserveDialLatency(elapsed time.Duration) { - a.latencies.WithLabelValues().Observe(elapsed.Seconds()) +func (s *ServerMetrics) ObserveDialLatency(elapsed time.Duration) { + s.endpointLatencies.WithLabelValues().Observe(elapsed.Seconds()) } -// ObserveFrontendWriteLatency records the latency of dial to the remote endpoint. -func (a *ServerMetrics) ObserveFrontendWriteLatency(elapsed time.Duration) { - a.frontendLatencies.WithLabelValues().Observe(elapsed.Seconds()) +// ObserveFrontendWriteLatency records the latency of blocking on stream send to the client. +func (s *ServerMetrics) ObserveFrontendWriteLatency(elapsed time.Duration) { + s.frontendLatencies.WithLabelValues().Observe(elapsed.Seconds()) } // ConnectionInc increments a new grpc client connection. -func (a *ServerMetrics) ConnectionInc(serviceMethod string) { - a.connections.With(prometheus.Labels{"service_method": serviceMethod}).Inc() +func (s *ServerMetrics) ConnectionInc(serviceMethod string) { + s.grpcConnections.With(prometheus.Labels{"service_method": serviceMethod}).Inc() } // ConnectionDec decrements a finished grpc client connection. -func (a *ServerMetrics) ConnectionDec(serviceMethod string) { - a.connections.With(prometheus.Labels{"service_method": serviceMethod}).Dec() +func (s *ServerMetrics) ConnectionDec(serviceMethod string) { + s.grpcConnections.With(prometheus.Labels{"service_method": serviceMethod}).Dec() } // HTTPConnectionDec increments a new HTTP CONNECTION connection. -func (a *ServerMetrics) HTTPConnectionInc() { a.httpConnections.Inc() } +func (s *ServerMetrics) HTTPConnectionInc() { s.httpConnections.Inc() } // HTTPConnectionDec decrements a finished HTTP CONNECTION connection. -func (a *ServerMetrics) HTTPConnectionDec() { a.httpConnections.Dec() } +func (s *ServerMetrics) HTTPConnectionDec() { s.httpConnections.Dec() } // SetBackendCount sets the number of backend connection. -func (a *ServerMetrics) SetBackendCount(count int) { - a.backend.WithLabelValues().Set(float64(count)) +func (s *ServerMetrics) SetBackendCount(count int) { + s.backend.WithLabelValues().Set(float64(count)) } // SetPendingDialCount sets the number of pending dials. -func (a *ServerMetrics) SetPendingDialCount(count int) { - a.pendingDials.WithLabelValues().Set(float64(count)) +func (s *ServerMetrics) SetPendingDialCount(count int) { + s.pendingDials.WithLabelValues().Set(float64(count)) } // FullRecvChannel retrieves the metric for counting full receive channels. -func (a *ServerMetrics) FullRecvChannel(serviceMethod string) prometheus.Gauge { - return a.fullRecvChannels.With(prometheus.Labels{"service_method": serviceMethod}) +func (s *ServerMetrics) FullRecvChannel(serviceMethod string) prometheus.Gauge { + return s.fullRecvChannels.With(prometheus.Labels{"service_method": serviceMethod}) } type DialFailureReason string @@ -225,6 +237,18 @@ const ( DialFailureFrontendClose DialFailureReason = "frontend_close" // Received a DIAL_CLS from the frontend before the dial completed. ) -func (a *ServerMetrics) ObserveDialFailure(reason DialFailureReason) { - a.dialFailures.With(prometheus.Labels{"reason": string(reason)}).Inc() +func (s *ServerMetrics) ObserveDialFailure(reason DialFailureReason) { + s.dialFailures.With(prometheus.Labels{"reason": string(reason)}).Inc() +} + +func (s *ServerMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { + commonmetrics.ObservePacket(s.streamPackets, segment, packetType) +} + +func (s *ServerMetrics) ObserveStreamErrorNoPacket(segment commonmetrics.Segment, err error) { + commonmetrics.ObserveStreamErrorNoPacket(s.streamErrors, segment, err) +} + +func (s *ServerMetrics) ObserveStreamError(segment commonmetrics.Segment, err error, packetType client.PacketType) { + commonmetrics.ObserveStreamError(s.streamErrors, segment, err, packetType) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 9b03971f9..59cbc2508 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -34,6 +34,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent" "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" @@ -64,11 +66,17 @@ const ( ) func (c *ProxyClientConnection) send(pkt *client.Packet) error { + const segment = commonmetrics.SegmentToClient + metrics.Metrics.ObservePacket(segment, pkt.Type) start := time.Now() defer metrics.Metrics.ObserveFrontendWriteLatency(time.Since(start)) if c.Mode == "grpc" { stream := c.Grpc - return stream.Send(pkt) + err := stream.Send(pkt) + if err != nil { + metrics.Metrics.ObserveStreamError(segment, err, pkt.Type) + } + return err } else if c.Mode == "http-connect" { if pkt.Type == client.PacketType_CLOSE_RSP { return c.CloseHTTP() @@ -396,7 +404,9 @@ func (s *ProxyServer) readFrontendToChannel(stream client.ProxyService_ProxyServ close(stopCh) return } + const segment = commonmetrics.SegmentFromClient if err != nil { + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) if status.Code(err) == codes.Canceled { klog.V(2).InfoS("Stream read from frontend cancelled", "userAgent", userAgent) } else { @@ -406,6 +416,7 @@ func (s *ProxyServer) readFrontendToChannel(stream client.ProxyService_ProxyServ close(stopCh) return } + metrics.Metrics.ObservePacket(segment, in.Type) select { case recvCh <- in: // Send didn't block, carry on. @@ -451,7 +462,10 @@ func (s *ProxyServer) serveRecvFrontend(stream client.ProxyService_ProxyServer, }, }, } + const segment = commonmetrics.SegmentToClient + metrics.Metrics.ObservePacket(segment, resp.Type) if err := stream.Send(resp); err != nil { + metrics.Metrics.ObserveStreamError(segment, err, resp.Type) klog.V(5).InfoS("Failed to send DIAL_RSP for no backend", "error", err, "dialID", random) } // The Dial is failing; no reason to keep this goroutine. @@ -713,12 +727,15 @@ func (s *ProxyServer) readBackendToChannel(stream agent.AgentService_ConnectServ close(stopCh) return } + const segment = commonmetrics.SegmentFromAgent if err != nil { + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) klog.ErrorS(err, "Receive stream from agent read failure") stopCh <- err close(stopCh) return } + metrics.Metrics.ObservePacket(segment, in.Type) select { case recvCh <- in: // Send didn't block, carry on. @@ -871,6 +888,7 @@ func (s *ProxyServer) serveRecvBackend(backend Backend, stream agent.AgentServic } klog.V(5).InfoS("Close backend of agent", "backend", stream, "agentID", agentID) } + func (s *ProxyServer) sendCloseRequest(stream agent.AgentService_ConnectServer, connectID int64, random int64, failMsg string) { pkt := &client.Packet{ Type: client.PacketType_CLOSE_REQ, @@ -880,7 +898,10 @@ func (s *ProxyServer) sendCloseRequest(stream agent.AgentService_ConnectServer, }, }, } + const segment = commonmetrics.SegmentToAgent + metrics.Metrics.ObservePacket(segment, pkt.Type) if err := stream.Send(pkt); err != nil { + metrics.Metrics.ObserveStreamError(segment, err, pkt.Type) klog.V(5).ErrorS(err, failMsg, "dialID", random, "agentID", agentID, "connectionID", connectID) } } diff --git a/pkg/testing/metrics/metrics.go b/pkg/testing/metrics/metrics.go index cabc9e98e..a7501390b 100644 --- a/pkg/testing/metrics/metrics.go +++ b/pkg/testing/metrics/metrics.go @@ -22,29 +22,47 @@ import ( "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" + agent "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" server "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" ) const ( - dialFailureHeader = ` + serverDialFailureHeader = ` # HELP konnectivity_network_proxy_server_dial_failure_count Number of dial failures observed. Multiple failures can occur for a single dial request. # TYPE konnectivity_network_proxy_server_dial_failure_count counter` - dialFailureSample = `konnectivity_network_proxy_server_dial_failure_count{reason="%s"} %d` + serverDialFailureSample = `konnectivity_network_proxy_server_dial_failure_count{reason="%s"} %d` + + agentDialFailureHeader = ` +# HELP konnectivity_network_proxy_agent_endpoint_dial_failure_total Number of failures dialing the remote endpoint, by reason (example: timeout). +# TYPE konnectivity_network_proxy_agent_endpoint_dial_failure_total counter` + agentDialFailureSample = `konnectivity_network_proxy_agent_endpoint_dial_failure_total{reason="%s"} %d` ) -func ExpectDialFailures(expected map[server.DialFailureReason]int) error { - expect := dialFailureHeader + "\n" +func ExpectServerDialFailures(expected map[server.DialFailureReason]int) error { + expect := serverDialFailureHeader + "\n" + for r, v := range expected { + expect += fmt.Sprintf(serverDialFailureSample+"\n", r, v) + } + return ExpectMetric(server.Namespace, server.Subsystem, server.DialFailuresMetric, expect) +} + +func ExpectServerDialFailure(reason server.DialFailureReason, count int) error { + return ExpectServerDialFailures(map[server.DialFailureReason]int{reason: count}) +} + +func ExpectAgentDialFailures(expected map[agent.DialFailureReason]int) error { + expect := agentDialFailureHeader + "\n" for r, v := range expected { - expect += fmt.Sprintf(dialFailureSample+"\n", r, v) + expect += fmt.Sprintf(agentDialFailureSample+"\n", r, v) } - return ExpectMetric(server.Subsystem, server.DialFailuresMetric, expect) + return ExpectMetric(agent.Namespace, agent.Subsystem, "endpoint_dial_failure_total", expect) } -func ExpectDialFailure(reason server.DialFailureReason, count int) error { - return ExpectDialFailures(map[server.DialFailureReason]int{reason: count}) +func ExpectAgentDialFailure(reason agent.DialFailureReason, count int) error { + return ExpectAgentDialFailures(map[agent.DialFailureReason]int{reason: count}) } -func ExpectMetric(subsystem, name, expected string) error { - fqName := prometheus.BuildFQName(server.Namespace, subsystem, name) +func ExpectMetric(namespace, subsystem, name, expected string) error { + fqName := prometheus.BuildFQName(namespace, subsystem, name) return promtest.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected), fqName) } diff --git a/tests/proxy_test.go b/tests/proxy_test.go index 9c3ea276c..78bd8209e 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -38,8 +38,9 @@ import ( "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" clientproto "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" "sigs.k8s.io/apiserver-network-proxy/pkg/agent" + metricsagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" "sigs.k8s.io/apiserver-network-proxy/pkg/server" - "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" + metricsserver "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" metricstest "sigs.k8s.io/apiserver-network-proxy/pkg/testing/metrics" agentproto "sigs.k8s.io/apiserver-network-proxy/proto/agent" "sigs.k8s.io/apiserver-network-proxy/proto/header" @@ -185,10 +186,14 @@ func TestProxyHandleDialError_GRPC(t *testing.T) { t.Errorf("Unexpected error: %v", err) } - if err := metricstest.ExpectDialFailure(metrics.DialFailureErrorResponse, 1); err != nil { + if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureErrorResponse, 1); err != nil { t.Error(err) } - metrics.Metrics.Reset() // For clean shutdown. + if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureUnknown, 1); err != nil { + t.Error(err) + } + metricsserver.Metrics.Reset() // For clean shutdown. + metricsagent.Metrics.Reset() } func TestProxyHandle_DoneContext_GRPC(t *testing.T) { @@ -331,10 +336,11 @@ func TestProxyDial_RequestCancelled_GRPC(t *testing.T) { } }() - if err := metricstest.ExpectDialFailure(metrics.DialFailureFrontendClose, 1); err != nil { + if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureFrontendClose, 1); err != nil { t.Error(err) } - metrics.Metrics.Reset() // For clean shutdown. + metricsserver.Metrics.Reset() // For clean shutdown. + metricsagent.Metrics.Reset() } func TestProxyDial_AgentTimeout_GRPC(t *testing.T) { @@ -369,10 +375,14 @@ func TestProxyDial_AgentTimeout_GRPC(t *testing.T) { t.Errorf("Unexpected error: %v", err) } - if err := metricstest.ExpectDialFailure(metrics.DialFailureErrorResponse, 1); err != nil { + if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureErrorResponse, 1); err != nil { t.Error(err) } - metrics.Metrics.Reset() // For clean shutdown. + if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureTimeout, 1); err != nil { + t.Error(err) + } + metricsserver.Metrics.Reset() // For clean shutdown. + metricsagent.Metrics.Reset() select { case <-tunnel.Done(): @@ -623,10 +633,14 @@ func TestFailedDial_HTTPCONN(t *testing.T) { t.Errorf("while waiting for connection to be closed: %v", err) } - if err := metricstest.ExpectDialFailure(metrics.DialFailureErrorResponse, 1); err != nil { + if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureErrorResponse, 1); err != nil { t.Error(err) } - metrics.Metrics.Reset() // For clean shutdown. + if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureUnknown, 1); err != nil { + t.Error(err) + } + metricsserver.Metrics.Reset() // For clean shutdown. + metricsagent.Metrics.Reset() } func localAddr(addr net.Addr) string { @@ -820,18 +834,26 @@ func waitForConnectedAgentCount(t testing.TB, expectedAgentCount int, proxy *ser } } -func assertNoDialFailures(t testing.TB) { +func assertNoServerDialFailures(t testing.TB) { t.Helper() - if err := metricstest.ExpectDialFailures(nil); err != nil { - t.Errorf("Unexpected %s metric: %v", metrics.DialFailuresMetric, err) + if err := metricstest.ExpectServerDialFailures(nil); err != nil { + t.Errorf("Unexpected %s metric: %v", metricsserver.DialFailuresMetric, err) } } +func assertNoAgentDialFailures(t testing.TB) { + t.Helper() + if err := metricstest.ExpectAgentDialFailures(nil); err != nil { + t.Errorf("Unexpected %s metric: %v", "endpoint_dial_failure_total", err) + } +} func expectCleanShutdown(t testing.TB) { - metrics.Metrics.Reset() + metricsserver.Metrics.Reset() + metricsagent.Metrics.Reset() currentGoRoutines := goleak.IgnoreCurrent() t.Cleanup(func() { goleak.VerifyNone(t, currentGoRoutines) - assertNoDialFailures(t) + assertNoServerDialFailures(t) + assertNoAgentDialFailures(t) }) } From f3708b7175268807e6f6b0bc0d931fe90d80761c Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Thu, 24 Nov 2022 16:55:37 +0100 Subject: [PATCH 13/24] Fix segfault on empty packet Verify if pkt is nil before logging the type. --- pkg/agent/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/agent/client.go b/pkg/agent/client.go index 51a0b378d..61aa882bd 100644 --- a/pkg/agent/client.go +++ b/pkg/agent/client.go @@ -384,13 +384,12 @@ func (a *Client) Serve() { return } - klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type) - if pkt == nil { klog.V(3).InfoS("empty packet received") continue } + klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type) metrics.Metrics.ObservePacket(commonmetrics.SegmentToAgent, pkt.Type) switch pkt.Type { case client.PacketType_DIAL_REQ: From bfa285eda65819424b3f6620c026fd791534e7ac Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Mon, 12 Dec 2022 23:06:47 -0800 Subject: [PATCH 14/24] konnectivity-client: make release-0.0 branch compatible with older k/k minor versions. --- konnectivity-client/go.mod | 19 +++++++------ konnectivity-client/go.sum | 57 ++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 42 deletions(-) diff --git a/konnectivity-client/go.mod b/konnectivity-client/go.mod index a29c22342..80fe85943 100644 --- a/konnectivity-client/go.mod +++ b/konnectivity-client/go.mod @@ -7,22 +7,23 @@ go 1.17 require ( github.com/golang/protobuf v1.4.3 github.com/prometheus/client_golang v1.0.0 - go.uber.org/goleak v1.2.0 + go.uber.org/goleak v1.1.10 google.golang.org/grpc v1.27.1 k8s.io/klog/v2 v2.0.0 ) require ( - github.com/beorn7/perks v1.0.1 // indirect + github.com/beorn7/perks v1.0.0 // indirect github.com/go-logr/logr v0.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect github.com/prometheus/common v0.4.1 // indirect - github.com/prometheus/procfs v0.1.3 // indirect - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect - golang.org/x/text v0.3.3 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect - google.golang.org/protobuf v1.26.0-rc.1 // indirect + github.com/prometheus/procfs v0.0.2 // indirect + golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a // indirect + golang.org/x/text v0.3.0 // indirect + golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 // indirect + google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect + google.golang.org/protobuf v1.23.0 // indirect ) diff --git a/konnectivity-client/go.sum b/konnectivity-client/go.sum index 4236926e1..373f9a9aa 100644 --- a/konnectivity-client/go.sum +++ b/konnectivity-client/go.sum @@ -3,9 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -30,19 +29,20 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -61,17 +61,17 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -79,50 +79,42 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -130,15 +122,14 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= From 2b43421a0d0848d62570a50ded787f34e1a0ee52 Mon Sep 17 00:00:00 2001 From: Zach Zhu Date: Fri, 9 Dec 2022 14:38:04 +0800 Subject: [PATCH 15/24] update .gitignore --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 2f5ff94b5..4e4b7c08d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,10 @@ /easy-rsa-master/ /easy-rsa.tar.gz /easy-rsa + +# editor and IDE paraphernalia +.idea +.vscode + +# macOS paraphernalia +.DS_Store From 6e6b6206b05abdfc37ac30a6ea5af0f53bc130af Mon Sep 17 00:00:00 2001 From: Zach Zhu Date: Fri, 9 Dec 2022 14:46:01 +0800 Subject: [PATCH 16/24] =?UTF-8?q?add=20more=20pprof=20functions=20to=20ser?= =?UTF-8?q?ver=20and=20agent=E2=80=99s=20admin=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/agent/app/server.go | 3 +++ cmd/server/app/server.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/cmd/agent/app/server.go b/cmd/agent/app/server.go index 1fa9411e9..d0a13a7ef 100644 --- a/cmd/agent/app/server.go +++ b/cmd/agent/app/server.go @@ -152,6 +152,9 @@ func (a *Agent) runAdminServer(o *options.GrpcProxyAgentOptions) error { if o.EnableProfiling { muxHandler.HandleFunc("/debug/pprof", util.RedirectTo("/debug/pprof/")) muxHandler.HandleFunc("/debug/pprof/", pprof.Index) + muxHandler.HandleFunc("/debug/pprof/profile", pprof.Profile) + muxHandler.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + muxHandler.HandleFunc("/debug/pprof/trace", pprof.Trace) if o.EnableContentionProfiling { runtime.SetBlockProfileRate(1) } diff --git a/cmd/server/app/server.go b/cmd/server/app/server.go index 71075124d..6302eecf2 100644 --- a/cmd/server/app/server.go +++ b/cmd/server/app/server.go @@ -391,6 +391,9 @@ func (p *Proxy) runAdminServer(o *options.ProxyRunOptions, server *server.ProxyS if o.EnableProfiling { muxHandler.HandleFunc("/debug/pprof", util.RedirectTo("/debug/pprof/")) muxHandler.HandleFunc("/debug/pprof/", netpprof.Index) + muxHandler.HandleFunc("/debug/pprof/profile", netpprof.Profile) + muxHandler.HandleFunc("/debug/pprof/symbol", netpprof.Symbol) + muxHandler.HandleFunc("/debug/pprof/trace", netpprof.Trace) if o.EnableContentionProfiling { runtime.SetBlockProfileRate(1) } From 005c7813d792a7228f83c8816e548cdd6ec839b2 Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Tue, 13 Dec 2022 10:32:18 -0800 Subject: [PATCH 17/24] konnectivity-client: add dial failures metric by reason. --- konnectivity-client/pkg/client/client.go | 44 +++++------- konnectivity-client/pkg/client/client_test.go | 72 ++++++++++++++----- .../pkg/client/metrics/metrics.go | 47 +++++++++++- .../pkg/common/metrics/testing/metrics.go | 50 +++++++++++++ tests/main_test.go | 4 ++ tests/proxy_test.go | 41 ++++++++--- 6 files changed, 201 insertions(+), 57 deletions(-) create mode 100644 konnectivity-client/pkg/common/metrics/testing/metrics.go diff --git a/konnectivity-client/pkg/client/client.go b/konnectivity-client/pkg/client/client.go index ec6489338..75314ebda 100644 --- a/konnectivity-client/pkg/client/client.go +++ b/konnectivity-client/pkg/client/client.go @@ -248,7 +248,7 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { result := dialResult{connid: resp.ConnectID} if resp.Error != "" { - result.err = &dialFailure{resp.Error, DialFailureEndpoint} + result.err = &dialFailure{resp.Error, metrics.DialFailureEndpoint} } select { // try to send to the result channel @@ -283,7 +283,7 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { klog.V(1).InfoS("DIAL_CLS after dial finished", "dialID", resp.Random) } else { result := dialResult{ - err: &dialFailure{"dial closed", DialFailureDialClosed}, + err: &dialFailure{"dial closed", metrics.DialFailureDialClosed}, } select { case pendingDial.resultCh <- result: @@ -336,6 +336,15 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { // Dial connects to the address on the named network, similar to // what net.Dial does. The only supported protocol is tcp. func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) { + conn, err := t.dialContext(requestCtx, protocol, address) + if err != nil { + _, reason := GetDialFailureReason(err) + metrics.Metrics.ObserveDialFailure(reason) + } + return conn, err +} + +func (t *grpcTunnel) dialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) { select { case <-t.done: return nil, errors.New("tunnel is closed") @@ -398,14 +407,14 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s case <-time.After(30 * time.Second): klog.V(5).InfoS("Timed out waiting for DialResp", "dialID", random) go t.closeDial(random) - return nil, &dialFailure{"dial timeout, backstop", DialFailureTimeout} + return nil, &dialFailure{"dial timeout, backstop", metrics.DialFailureTimeout} case <-requestCtx.Done(): klog.V(5).InfoS("Context canceled waiting for DialResp", "ctxErr", requestCtx.Err(), "dialID", random) go t.closeDial(random) - return nil, &dialFailure{"dial timeout, context", DialFailureContext} + return nil, &dialFailure{"dial timeout, context", metrics.DialFailureContext} case <-t.done: klog.V(5).InfoS("Tunnel closed while waiting for DialResp", "dialID", random) - return nil, &dialFailure{"tunnel closed", DialFailureTunnelClosed} + return nil, &dialFailure{"tunnel closed", metrics.DialFailureTunnelClosed} } return c, nil @@ -443,38 +452,19 @@ func (t *grpcTunnel) isClosing() bool { return atomic.LoadUint32(&t.closing) != 0 } -func GetDialFailureReason(err error) (isDialFailure bool, reason DialFailureReason) { +func GetDialFailureReason(err error) (isDialFailure bool, reason metrics.DialFailureReason) { var df *dialFailure if errors.As(err, &df) { return true, df.reason } - return false, DialFailureUnknown + return false, metrics.DialFailureUnknown } type dialFailure struct { msg string - reason DialFailureReason + reason metrics.DialFailureReason } func (df *dialFailure) Error() string { return df.msg } - -type DialFailureReason string - -const ( - DialFailureUnknown DialFailureReason = "unknown" - // DialFailureTimeout indicates the hard 30 second timeout was hit. - DialFailureTimeout DialFailureReason = "timeout" - // DialFailureContext indicates that the context was cancelled or reached it's deadline before - // the dial response was returned. - DialFailureContext DialFailureReason = "context" - // DialFailureEndpoint indicates that the konnectivity-agent was unable to reach the backend endpoint. - DialFailureEndpoint DialFailureReason = "endpoint" - // DialFailureDialClosed indicates that the client received a CloseDial response, indicating the - // connection was closed before the dial could complete. - DialFailureDialClosed DialFailureReason = "dialclosed" - // DialFailureTunnelClosed indicates that the client connection was closed before the dial could - // complete. - DialFailureTunnelClosed DialFailureReason = "tunnelclosed" -) diff --git a/konnectivity-client/pkg/client/client_test.go b/konnectivity-client/pkg/client/client_test.go index e5de029c7..474049411 100644 --- a/konnectivity-client/pkg/client/client_test.go +++ b/konnectivity-client/pkg/client/client_test.go @@ -30,6 +30,8 @@ import ( "go.uber.org/goleak" "google.golang.org/grpc" "k8s.io/klog/v2" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + metricstest "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics/testing" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) @@ -37,12 +39,13 @@ func TestMain(m *testing.M) { fs := flag.NewFlagSet("test", flag.PanicOnError) klog.InitFlags(fs) fs.Set("v", "9") + metrics.Metrics.RegisterMetrics(prometheus.DefaultRegisterer) m.Run() } func TestDial(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -68,12 +71,16 @@ func TestDial(t *testing.T) { if ts.packets[0].GetDialRequest().Address != "127.0.0.1:80" { t.Errorf("expect packet.address %v; got %v", "127.0.0.1:80", ts.packets[0].GetDialRequest().Address) } + + if err := metricstest.ExpectClientDialFailures(nil); err != nil { + t.Error(err) + } } // TestDialRace exercises the scenario where serve() observes and handles DIAL_RSP // before DialContext() does any work after sending the DIAL_REQ. func TestDialRace(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -119,7 +126,7 @@ func (s fakeSlowSend) Send(p *client.Packet) error { } func TestData(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -175,7 +182,7 @@ func TestData(t *testing.T) { } func TestClose(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -210,7 +217,7 @@ func TestCloseTimeout(t *testing.T) { if testing.Short() { t.Skip() } - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx := context.Background() s, ps := pipe() @@ -274,7 +281,7 @@ func TestCreateSingleUseGrpcTunnelWithContext_NoLeakOnFailure(t *testing.T) { } func TestDialAfterTunnelCancelled(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -293,6 +300,8 @@ func TestDialAfterTunnelCancelled(t *testing.T) { if err == nil { t.Fatalf("expect err when dialing after tunnel closed") } + // TODO(jkh52): verify dial failure metric. + metrics.Metrics.Reset() // For clean shutdown. select { case <-tunnel.Done(): @@ -302,7 +311,7 @@ func TestDialAfterTunnelCancelled(t *testing.T) { } func TestDial_RequestContextCancelled(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) s, ps := pipe() defer ps.Close() @@ -322,9 +331,6 @@ func TestDial_RequestContextCancelled(t *testing.T) { go ts.serve() func() { - // Tunnel should be shut down when the dial fails. - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) - tunnel := newUnstartedTunnel(s, s.conn()) go tunnel.serve(context.Background()) @@ -336,10 +342,15 @@ func TestDial_RequestContextCancelled(t *testing.T) { isDialFailure, reason := GetDialFailureReason(err) if !isDialFailure { t.Errorf("Unexpected non-dial failure error: %v", err) - } else if reason != DialFailureContext { + } else if reason != metrics.DialFailureContext { t.Errorf("Expected DialFailureContext, got %v", reason) } + if err := metricstest.ExpectClientDialFailure(metrics.DialFailureContext, 1); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. + ts.assertPacketType(0, client.PacketType_DIAL_REQ) waitForDialClsStart := time.Now() select { @@ -361,7 +372,7 @@ func TestDial_RequestContextCancelled(t *testing.T) { } func TestDial_BackendError(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) s, ps := pipe() ts := testServer(ps, 100) @@ -393,15 +404,20 @@ func TestDial_BackendError(t *testing.T) { isDialFailure, reason := GetDialFailureReason(err) if !isDialFailure { t.Errorf("Unexpected non-dial failure error: %v", err) - } else if reason != DialFailureEndpoint { + } else if reason != metrics.DialFailureEndpoint { t.Errorf("Expected DialFailureEndpoint, got %v", reason) } ts.assertPacketType(0, client.PacketType_DIAL_REQ) + + if err := metricstest.ExpectClientDialFailure(metrics.DialFailureEndpoint, 1); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. } func TestDial_Closed(t *testing.T) { - defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + expectCleanShutdown(t) s, ps := pipe() defer ps.Close() @@ -435,12 +451,17 @@ func TestDial_Closed(t *testing.T) { isDialFailure, reason := GetDialFailureReason(err) if !isDialFailure { t.Errorf("Unexpected non-dial failure error: %v", err) - } else if reason != DialFailureDialClosed { + } else if reason != metrics.DialFailureDialClosed { t.Errorf("Expected DialFailureDialClosed, got %v", reason) } ts.assertPacketType(0, client.PacketType_DIAL_REQ) + if err := metricstest.ExpectClientDialFailure(metrics.DialFailureDialClosed, 1); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. + select { case <-tunnel.Done(): case <-time.After(30 * time.Second): @@ -450,11 +471,11 @@ func TestDial_Closed(t *testing.T) { } func TestRegisterMetrics(t *testing.T) { - Metrics.RegisterMetrics(prometheus.DefaultRegisterer, "namespace", "subsystem") + Metrics.RegisterMetrics(prometheus.DefaultRegisterer) } func TestLegacyRegisterMetrics(t *testing.T) { - Metrics.LegacyRegisterMetrics(prometheus.MustRegister, "namespace", "subsystem") + Metrics.LegacyRegisterMetrics(prometheus.MustRegister) } // TODO: Move to common testing library @@ -643,3 +664,20 @@ func (s *proxyServer) handleData(pkt *client.Packet) *client.Packet { }, } } + +func assertNoClientDialFailures(t testing.TB) { + t.Helper() + if err := metricstest.ExpectClientDialFailures(nil); err != nil { + t.Errorf("Unexpected %s metric: %v", "dial_failure_total", err) + } +} + +func expectCleanShutdown(t testing.TB) { + metrics.Metrics.Reset() + currentGoRoutines := goleak.IgnoreCurrent() + t.Cleanup(func() { + goleak.VerifyNone(t, currentGoRoutines) + assertNoClientDialFailures(t) + metrics.Metrics.Reset() + }) +} diff --git a/konnectivity-client/pkg/client/metrics/metrics.go b/konnectivity-client/pkg/client/metrics/metrics.go index afef1c772..a23625236 100644 --- a/konnectivity-client/pkg/client/metrics/metrics.go +++ b/konnectivity-client/pkg/client/metrics/metrics.go @@ -41,29 +41,67 @@ type ClientMetrics struct { registerOnce sync.Once streamPackets *prometheus.CounterVec streamErrors *prometheus.CounterVec + dialFailures *prometheus.CounterVec } +type DialFailureReason string + +const ( + DialFailureUnknown DialFailureReason = "unknown" + // DialFailureTimeout indicates the hard 30 second timeout was hit. + DialFailureTimeout DialFailureReason = "timeout" + // DialFailureContext indicates that the context was cancelled or reached it's deadline before + // the dial response was returned. + DialFailureContext DialFailureReason = "context" + // DialFailureEndpoint indicates that the konnectivity-agent was unable to reach the backend endpoint. + DialFailureEndpoint DialFailureReason = "endpoint" + // DialFailureDialClosed indicates that the client received a CloseDial response, indicating the + // connection was closed before the dial could complete. + DialFailureDialClosed DialFailureReason = "dialclosed" + // DialFailureTunnelClosed indicates that the client connection was closed before the dial could + // complete. + DialFailureTunnelClosed DialFailureReason = "tunnelclosed" +) + func newMetrics() *ClientMetrics { + // The denominator (total dials started) for both + // dial_failure_total and dial_duration_seconds is the + // stream_packets_total (common metric), where segment is + // "from_client" and packet_type is "DIAL_REQ". + dialFailures := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "dial_failure_total", + Help: "Number of dial failures observed, by reason (example: remote endpoint error)", + }, + []string{ + "reason", + }, + ) return &ClientMetrics{ streamPackets: commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem), streamErrors: commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem), + dialFailures: dialFailures, } } // RegisterMetrics registers all metrics with the client application. -func (c *ClientMetrics) RegisterMetrics(r prometheus.Registerer, namespace, subsystem string) { +func (c *ClientMetrics) RegisterMetrics(r prometheus.Registerer) { c.registerOnce.Do(func() { r.MustRegister(c.streamPackets) r.MustRegister(c.streamErrors) + r.MustRegister(c.dialFailures) }) } // LegacyRegisterMetrics registers all metrics via MustRegister func. // TODO: remove this once https://github.com/kubernetes/kubernetes/pull/114293 is available. -func (c *ClientMetrics) LegacyRegisterMetrics(mustRegisterFn func(...prometheus.Collector), namespace, subsystem string) { +func (c *ClientMetrics) LegacyRegisterMetrics(mustRegisterFn func(...prometheus.Collector)) { c.registerOnce.Do(func() { mustRegisterFn(c.streamPackets) mustRegisterFn(c.streamErrors) + mustRegisterFn(c.dialFailures) }) } @@ -71,6 +109,11 @@ func (c *ClientMetrics) LegacyRegisterMetrics(mustRegisterFn func(...prometheus. func (c *ClientMetrics) Reset() { c.streamPackets.Reset() c.streamErrors.Reset() + c.dialFailures.Reset() +} + +func (c *ClientMetrics) ObserveDialFailure(reason DialFailureReason) { + c.dialFailures.WithLabelValues(string(reason)).Inc() } func (c *ClientMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { diff --git a/konnectivity-client/pkg/common/metrics/testing/metrics.go b/konnectivity-client/pkg/common/metrics/testing/metrics.go new file mode 100644 index 000000000..5b015f1d1 --- /dev/null +++ b/konnectivity-client/pkg/common/metrics/testing/metrics.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "strings" + + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" +) + +const ( + clientDialFailureHeader = ` +# HELP konnectivity_network_proxy_client_dial_failure_total Number of dial failures observed, by reason (example: remote endpoint error) +# TYPE konnectivity_network_proxy_client_dial_failure_total counter` + clientDialFailureSample = `konnectivity_network_proxy_client_dial_failure_total{reason="%s"} %d` +) + +func ExpectClientDialFailures(expected map[metrics.DialFailureReason]int) error { + expect := clientDialFailureHeader + "\n" + for r, v := range expected { + expect += fmt.Sprintf(clientDialFailureSample+"\n", r, v) + } + return ExpectMetric(metrics.Namespace, metrics.Subsystem, "dial_failure_total", expect) +} + +func ExpectClientDialFailure(reason metrics.DialFailureReason, count int) error { + return ExpectClientDialFailures(map[metrics.DialFailureReason]int{reason: count}) +} + +func ExpectMetric(namespace, subsystem, name, expected string) error { + fqName := prometheus.BuildFQName(namespace, subsystem, name) + return promtest.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected), fqName) +} diff --git a/tests/main_test.go b/tests/main_test.go index 1df546080..efc51a166 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -20,13 +20,17 @@ import ( "flag" "testing" + "github.com/prometheus/client_golang/prometheus" "k8s.io/klog/v2" + + metricsclient "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" ) func TestMain(m *testing.M) { fs := flag.NewFlagSet("mock-flags", flag.PanicOnError) klog.InitFlags(fs) fs.Set("v", "1") // Set klog verbosity. + metricsclient.Metrics.RegisterMetrics(prometheus.DefaultRegisterer) m.Run() } diff --git a/tests/proxy_test.go b/tests/proxy_test.go index 78bd8209e..99055ecda 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -36,6 +36,9 @@ import ( "google.golang.org/grpc/metadata" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + metricsclient "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + clientmetricstest "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics/testing" clientproto "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" "sigs.k8s.io/apiserver-network-proxy/pkg/agent" metricsagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics" @@ -192,8 +195,7 @@ func TestProxyHandleDialError_GRPC(t *testing.T) { if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureUnknown, 1); err != nil { t.Error(err) } - metricsserver.Metrics.Reset() // For clean shutdown. - metricsagent.Metrics.Reset() + resetAllMetrics() // For clean shutdown. } func TestProxyHandle_DoneContext_GRPC(t *testing.T) { @@ -325,7 +327,7 @@ func TestProxyDial_RequestCancelled_GRPC(t *testing.T) { _, err = tunnel.DialContext(ctx, "tcp", blackhole) if err == nil { t.Error("Expected error when context is cancelled, did not receive error") - } else if _, reason := client.GetDialFailureReason(err); reason != client.DialFailureContext { + } else if _, reason := client.GetDialFailureReason(err); reason != metricsclient.DialFailureContext { t.Errorf("Unexpected error: %v", err) } @@ -336,11 +338,13 @@ func TestProxyDial_RequestCancelled_GRPC(t *testing.T) { } }() + if err := clientmetricstest.ExpectClientDialFailure(metrics.DialFailureContext, 1); err != nil { + t.Error(err) + } if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureFrontendClose, 1); err != nil { t.Error(err) } - metricsserver.Metrics.Reset() // For clean shutdown. - metricsagent.Metrics.Reset() + resetAllMetrics() // For clean shutdown. } func TestProxyDial_AgentTimeout_GRPC(t *testing.T) { @@ -371,18 +375,20 @@ func TestProxyDial_AgentTimeout_GRPC(t *testing.T) { _, err = tunnel.DialContext(context.Background(), "tcp", blackhole) if err == nil { t.Error("Expected error when context is cancelled, did not receive error") - } else if _, reason := client.GetDialFailureReason(err); reason != client.DialFailureEndpoint { + } else if _, reason := client.GetDialFailureReason(err); reason != metricsclient.DialFailureEndpoint { t.Errorf("Unexpected error: %v", err) } + if err := clientmetricstest.ExpectClientDialFailure(metrics.DialFailureEndpoint, 1); err != nil { + t.Error(err) + } if err := metricstest.ExpectServerDialFailure(metricsserver.DialFailureErrorResponse, 1); err != nil { t.Error(err) } if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureTimeout, 1); err != nil { t.Error(err) } - metricsserver.Metrics.Reset() // For clean shutdown. - metricsagent.Metrics.Reset() + resetAllMetrics() // For clean shutdown. select { case <-tunnel.Done(): @@ -639,8 +645,7 @@ func TestFailedDial_HTTPCONN(t *testing.T) { if err := metricstest.ExpectAgentDialFailure(metricsagent.DialFailureUnknown, 1); err != nil { t.Error(err) } - metricsserver.Metrics.Reset() // For clean shutdown. - metricsagent.Metrics.Reset() + resetAllMetrics() // For clean shutdown. } func localAddr(addr net.Addr) string { @@ -834,6 +839,13 @@ func waitForConnectedAgentCount(t testing.TB, expectedAgentCount int, proxy *ser } } +func assertNoClientDialFailures(t testing.TB) { + t.Helper() + if err := clientmetricstest.ExpectClientDialFailures(nil); err != nil { + t.Errorf("Unexpected %s metric: %v", "dial_failure_total", err) + } +} + func assertNoServerDialFailures(t testing.TB) { t.Helper() if err := metricstest.ExpectServerDialFailures(nil); err != nil { @@ -847,12 +859,19 @@ func assertNoAgentDialFailures(t testing.TB) { t.Errorf("Unexpected %s metric: %v", "endpoint_dial_failure_total", err) } } -func expectCleanShutdown(t testing.TB) { + +func resetAllMetrics() { + metricsclient.Metrics.Reset() metricsserver.Metrics.Reset() metricsagent.Metrics.Reset() +} + +func expectCleanShutdown(t testing.TB) { + resetAllMetrics() currentGoRoutines := goleak.IgnoreCurrent() t.Cleanup(func() { goleak.VerifyNone(t, currentGoRoutines) + assertNoClientDialFailures(t) assertNoServerDialFailures(t) assertNoAgentDialFailures(t) }) From 4784bcee08938bcbb29b57cad32f5c398c562d29 Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Tue, 20 Dec 2022 14:08:32 -0800 Subject: [PATCH 18/24] Fix broken `make lint`. Replace deprecated `golint` with `revive`. Upgrade linter to latest version. Apply lint fixes found. --- Makefile | 4 ++-- cmd/agent/app/options/options_test.go | 6 +++--- cmd/server/app/options/options_test.go | 6 +++--- cmd/server/app/server.go | 7 ++++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index ff2a65ccc..5017b0eaf 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ endif GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) INSTALL_LOCATION:=$(shell go env GOPATH)/bin -GOLANGCI_LINT_VERSION ?= 1.45.2 +GOLANGCI_LINT_VERSION ?= 1.50.1 GOSEC_VERSION ?= 2.13.1 REGISTRY ?= gcr.io/$(shell gcloud config get-value project) @@ -88,7 +88,7 @@ bin/proxy-server: proto/agent/agent.pb.go konnectivity-client/proto/client/clien .PHONY: lint lint: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(INSTALL_LOCATION) v$(GOLANGCI_LINT_VERSION) - $(INSTALL_LOCATION)/golangci-lint run --no-config --disable-all --enable=gofmt,golint,gosec,govet,unused --fix --verbose --timeout 3m + $(INSTALL_LOCATION)/golangci-lint run --no-config --disable-all --enable=gofmt,revive,gosec,govet,unused --fix --verbose --timeout 3m ## -------------------------------------- ## Go diff --git a/cmd/agent/app/options/options_test.go b/cmd/agent/app/options/options_test.go index bb830a06b..bce6fe933 100644 --- a/cmd/agent/app/options/options_test.go +++ b/cmd/agent/app/options/options_test.go @@ -17,11 +17,11 @@ limitations under the License. package options import ( - "testing" + "fmt" + "github.com/stretchr/testify/assert" "reflect" + "testing" "time" - "github.com/stretchr/testify/assert" - "fmt" ) /* diff --git a/cmd/server/app/options/options_test.go b/cmd/server/app/options/options_test.go index aa99e36ed..0598bb977 100644 --- a/cmd/server/app/options/options_test.go +++ b/cmd/server/app/options/options_test.go @@ -17,11 +17,11 @@ limitations under the License. package options import ( - "testing" - "reflect" - "time" "fmt" "github.com/stretchr/testify/assert" + "reflect" + "testing" + "time" ) /* diff --git a/cmd/server/app/server.go b/cmd/server/app/server.go index 6302eecf2..7073c7c15 100644 --- a/cmd/server/app/server.go +++ b/cmd/server/app/server.go @@ -399,9 +399,10 @@ func (p *Proxy) runAdminServer(o *options.ProxyRunOptions, server *server.ProxyS } } adminServer := &http.Server{ - Addr: net.JoinHostPort(o.AdminBindAddress, strconv.Itoa(o.AdminPort)), - Handler: muxHandler, - MaxHeaderBytes: 1 << 20, + Addr: net.JoinHostPort(o.AdminBindAddress, strconv.Itoa(o.AdminPort)), + Handler: muxHandler, + MaxHeaderBytes: 1 << 20, + ReadHeaderTimeout: ReadHeaderTimeout, } labels := runpprof.Labels( From cce2a51b373023acdaa1becfe3d28ecf014a0a91 Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Tue, 20 Dec 2022 19:46:25 +0000 Subject: [PATCH 19/24] konnectivity-client: add a tunnels gauge metric. --- konnectivity-client/pkg/client/client.go | 41 ++++++++++++++++- konnectivity-client/pkg/client/client_test.go | 46 +++++++++++++++++++ .../pkg/client/metrics/metrics.go | 33 +++++++++++++ .../pkg/common/metrics/testing/metrics.go | 17 +++++++ 4 files changed, 136 insertions(+), 1 deletion(-) diff --git a/konnectivity-client/pkg/client/client.go b/konnectivity-client/pkg/client/client.go index 75314ebda..cb186cefc 100644 --- a/konnectivity-client/pkg/client/client.go +++ b/konnectivity-client/pkg/client/client.go @@ -134,6 +134,9 @@ type grpcTunnel struct { // closing should only be accessed through atomic methods. // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+ closing uint32 + + // Stores the current metrics.ClientConnectionStatus + prevStatus atomic.Value } type clientConn interface { @@ -185,7 +188,7 @@ func CreateSingleUseGrpcTunnelWithContext(createCtx, tunnelCtx context.Context, } func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *grpcTunnel { - return &grpcTunnel{ + t := grpcTunnel{ stream: stream, clientConn: c, pendingDial: pendingDialManager{pendingDials: make(map[int64]pendingDial)}, @@ -193,6 +196,36 @@ func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *g readTimeoutSeconds: 10, done: make(chan struct{}), } + s := metrics.ClientConnectionStatusCreated + t.prevStatus.Store(s) + metrics.Metrics.GetClientConnectionsMetric().WithLabelValues(string(s)).Inc() + return &t +} + +func (t *grpcTunnel) updateMetric(status metrics.ClientConnectionStatus) { + select { + case <-t.Done(): + return + default: + } + + prevStatus := t.prevStatus.Swap(status).(metrics.ClientConnectionStatus) + + m := metrics.Metrics.GetClientConnectionsMetric() + m.WithLabelValues(string(prevStatus)).Dec() + m.WithLabelValues(string(status)).Inc() +} + +// closeMetric should be called exactly once to finalize client_connections metric. +func (t *grpcTunnel) closeMetric() { + select { + case <-t.Done(): + return + default: + } + prevStatus := t.prevStatus.Load().(metrics.ClientConnectionStatus) + + metrics.Metrics.GetClientConnectionsMetric().WithLabelValues(string(prevStatus)).Dec() } func (t *grpcTunnel) serve(tunnelCtx context.Context) { @@ -204,6 +237,8 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { // close any channels remaining for these connections. t.conns.closeAll() + t.closeMetric() + close(t.done) }() @@ -249,6 +284,8 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context) { result := dialResult{connid: resp.ConnectID} if resp.Error != "" { result.err = &dialFailure{resp.Error, metrics.DialFailureEndpoint} + } else { + t.updateMetric(metrics.ClientConnectionStatusOk) } select { // try to send to the result channel @@ -355,6 +392,8 @@ func (t *grpcTunnel) dialContext(requestCtx context.Context, protocol, address s return nil, errors.New("protocol not supported") } + t.updateMetric(metrics.ClientConnectionStatusDialing) + random := rand.Int63() /* #nosec G404 */ // This channel is closed once we're returning and no longer waiting on resultCh diff --git a/konnectivity-client/pkg/client/client_test.go b/konnectivity-client/pkg/client/client_test.go index 474049411..3ac773b58 100644 --- a/konnectivity-client/pkg/client/client_test.go +++ b/konnectivity-client/pkg/client/client_test.go @@ -56,6 +56,10 @@ func TestDial(t *testing.T) { tunnel := newUnstartedTunnel(s, s.conn()) + if err := metricstest.ExpectClientConnection(metrics.ClientConnectionStatusCreated, 1); err != nil { + t.Error(err) + } + go tunnel.serve(ctx) go ts.serve() @@ -72,9 +76,17 @@ func TestDial(t *testing.T) { t.Errorf("expect packet.address %v; got %v", "127.0.0.1:80", ts.packets[0].GetDialRequest().Address) } + if err := metricstest.ExpectClientConnections(map[metrics.ClientConnectionStatus]int{ + metrics.ClientConnectionStatusCreated: 0, + metrics.ClientConnectionStatusDialing: 0, + metrics.ClientConnectionStatusOk: 1, + }); err != nil { + t.Error(err) + } if err := metricstest.ExpectClientDialFailures(nil); err != nil { t.Error(err) } + metrics.Metrics.Reset() // For clean shutdown. } // TestDialRace exercises the scenario where serve() observes and handles DIAL_RSP @@ -108,6 +120,10 @@ func TestDialRace(t *testing.T) { if ts.packets[0].GetDialRequest().Address != "127.0.0.1:80" { t.Errorf("expect packet.address %v; got %v", "127.0.0.1:80", ts.packets[0].GetDialRequest().Address) } + + if err := metricstest.ExpectClientDialFailures(nil); err != nil { + t.Error(err) + } } // fakeSlowSend wraps ProxyService_ProxyClient and adds an artificial 1 second delay after calling Send @@ -201,6 +217,14 @@ func TestClose(t *testing.T) { t.Fatalf("expect nil; got %v", err) } + if err := metricstest.ExpectClientConnections(map[metrics.ClientConnectionStatus]int{ + metrics.ClientConnectionStatusCreated: 0, + metrics.ClientConnectionStatusDialing: 0, + metrics.ClientConnectionStatusOk: 1, + }); err != nil { + t.Error(err) + } + if err := conn.Close(); err != nil { t.Error(err) } @@ -211,6 +235,16 @@ func TestClose(t *testing.T) { if ts.packets[1].GetCloseRequest().ConnectID != 100 { t.Errorf("expect connectID=100; got %d", ts.packets[1].GetCloseRequest().ConnectID) } + + <-tunnel.Done() + if err := metricstest.ExpectClientConnections(map[metrics.ClientConnectionStatus]int{ + metrics.ClientConnectionStatusCreated: 0, + metrics.ClientConnectionStatusDialing: 0, + metrics.ClientConnectionStatusOk: 0, + }); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. } func TestCloseTimeout(t *testing.T) { @@ -254,6 +288,15 @@ func TestCloseTimeout(t *testing.T) { t.Errorf("expected %v but got %v", errConnCloseTimeout, err) } + <-tunnel.Done() + if err := metricstest.ExpectClientConnections(map[metrics.ClientConnectionStatus]int{ + metrics.ClientConnectionStatusCreated: 0, + metrics.ClientConnectionStatusDialing: 0, + metrics.ClientConnectionStatusOk: 0, + }); err != nil { + t.Error(err) + } + metrics.Metrics.Reset() // For clean shutdown. } func TestCreateSingleUseGrpcTunnel_NoLeakOnFailure(t *testing.T) { @@ -300,6 +343,7 @@ func TestDialAfterTunnelCancelled(t *testing.T) { if err == nil { t.Fatalf("expect err when dialing after tunnel closed") } + // TODO(jkh52): verify dial failure metric. metrics.Metrics.Reset() // For clean shutdown. @@ -678,6 +722,8 @@ func expectCleanShutdown(t testing.TB) { t.Cleanup(func() { goleak.VerifyNone(t, currentGoRoutines) assertNoClientDialFailures(t) + // The tunnels gauge metric is intentionally not included here, to avoid + // brittle gauge assertions. Tests should directly verify it. metrics.Metrics.Reset() }) } diff --git a/konnectivity-client/pkg/client/metrics/metrics.go b/konnectivity-client/pkg/client/metrics/metrics.go index a23625236..03e9d94da 100644 --- a/konnectivity-client/pkg/client/metrics/metrics.go +++ b/konnectivity-client/pkg/client/metrics/metrics.go @@ -42,6 +42,7 @@ type ClientMetrics struct { streamPackets *prometheus.CounterVec streamErrors *prometheus.CounterVec dialFailures *prometheus.CounterVec + clientConns *prometheus.GaugeVec } type DialFailureReason string @@ -63,6 +64,19 @@ const ( DialFailureTunnelClosed DialFailureReason = "tunnelclosed" ) +type ClientConnectionStatus string + +const ( + // The connection is created but has not yet been dialed. + ClientConnectionStatusCreated ClientConnectionStatus = "created" + // The connection is pending dial response. + ClientConnectionStatusDialing ClientConnectionStatus = "dialing" + // The connection is established. + ClientConnectionStatusOk ClientConnectionStatus = "ok" + // The connection is closing. + ClientConnectionStatusClosing ClientConnectionStatus = "closing" +) + func newMetrics() *ClientMetrics { // The denominator (total dials started) for both // dial_failure_total and dial_duration_seconds is the @@ -79,10 +93,22 @@ func newMetrics() *ClientMetrics { "reason", }, ) + clientConns := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "client_connections", + Help: "Number of open client connections, by status (Example: dialing)", + }, + []string{ + "status", + }, + ) return &ClientMetrics{ streamPackets: commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem), streamErrors: commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem), dialFailures: dialFailures, + clientConns: clientConns, } } @@ -92,6 +118,7 @@ func (c *ClientMetrics) RegisterMetrics(r prometheus.Registerer) { r.MustRegister(c.streamPackets) r.MustRegister(c.streamErrors) r.MustRegister(c.dialFailures) + r.MustRegister(c.clientConns) }) } @@ -102,6 +129,7 @@ func (c *ClientMetrics) LegacyRegisterMetrics(mustRegisterFn func(...prometheus. mustRegisterFn(c.streamPackets) mustRegisterFn(c.streamErrors) mustRegisterFn(c.dialFailures) + mustRegisterFn(c.clientConns) }) } @@ -110,12 +138,17 @@ func (c *ClientMetrics) Reset() { c.streamPackets.Reset() c.streamErrors.Reset() c.dialFailures.Reset() + c.clientConns.Reset() } func (c *ClientMetrics) ObserveDialFailure(reason DialFailureReason) { c.dialFailures.WithLabelValues(string(reason)).Inc() } +func (c *ClientMetrics) GetClientConnectionsMetric() *prometheus.GaugeVec { + return c.clientConns +} + func (c *ClientMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { commonmetrics.ObservePacket(c.streamPackets, segment, packetType) } diff --git a/konnectivity-client/pkg/common/metrics/testing/metrics.go b/konnectivity-client/pkg/common/metrics/testing/metrics.go index 5b015f1d1..85f146484 100644 --- a/konnectivity-client/pkg/common/metrics/testing/metrics.go +++ b/konnectivity-client/pkg/common/metrics/testing/metrics.go @@ -30,6 +30,11 @@ const ( # HELP konnectivity_network_proxy_client_dial_failure_total Number of dial failures observed, by reason (example: remote endpoint error) # TYPE konnectivity_network_proxy_client_dial_failure_total counter` clientDialFailureSample = `konnectivity_network_proxy_client_dial_failure_total{reason="%s"} %d` + + clientConnsHeader = ` +# HELP konnectivity_network_proxy_client_client_connections Number of open client connections, by status (Example: dialing) +# TYPE konnectivity_network_proxy_client_client_connections gauge` + clientConnsSample = `konnectivity_network_proxy_client_client_connections{status="%s"} %d` ) func ExpectClientDialFailures(expected map[metrics.DialFailureReason]int) error { @@ -44,6 +49,18 @@ func ExpectClientDialFailure(reason metrics.DialFailureReason, count int) error return ExpectClientDialFailures(map[metrics.DialFailureReason]int{reason: count}) } +func ExpectClientConnections(expected map[metrics.ClientConnectionStatus]int) error { + expect := clientConnsHeader + "\n" + for r, v := range expected { + expect += fmt.Sprintf(clientConnsSample+"\n", r, v) + } + return ExpectMetric(metrics.Namespace, metrics.Subsystem, "client_connections", expect) +} + +func ExpectClientConnection(status metrics.ClientConnectionStatus, count int) error { + return ExpectClientConnections(map[metrics.ClientConnectionStatus]int{status: count}) +} + func ExpectMetric(namespace, subsystem, name, expected string) error { fqName := prometheus.BuildFQName(namespace, subsystem, name) return promtest.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected), fqName) From eecd9a1a9b4d71dd99e4faca0322e5868388906f Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 27 Dec 2022 13:29:51 +0100 Subject: [PATCH 20/24] Rename ApiServerMapping to APIServerMapping Rename variable to make linter happy. Fixes commit 5d769c17f70c (Extend agent to support traffic from cluster to master network.) --- cmd/agent/app/options/options.go | 30 +++++++++++++++--------------- cmd/agent/app/server.go | 4 ++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index 2bc53c0af..27517e896 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -79,9 +79,9 @@ type GrpcProxyAgentOptions struct { // The check is an "unlocked" read but is still use at your own peril. WarnOnChannelLimit bool - SyncForever bool - BindAddress string - ApiServerMapping portMapping + SyncForever bool + BindAddress string + APIServerMapping portMapping } var _ pflag.Value = &portMapping{} @@ -139,7 +139,7 @@ func (o *GrpcProxyAgentOptions) Flags() *pflag.FlagSet { flags.StringVar(&o.ServiceAccountTokenPath, "service-account-token-path", o.ServiceAccountTokenPath, "If non-empty proxy agent uses this token to prove its identity to the proxy server.") flags.StringVar(&o.AgentIdentifiers, "agent-identifiers", o.AgentIdentifiers, "Identifiers of the agent that will be used by the server when choosing agent. N.B. the list of identifiers must be in URL encoded format. e.g.,host=localhost&host=node1.mydomain.com&cidr=127.0.0.1/16&ipv4=1.2.3.4&ipv4=5.6.7.8&ipv6=:::::&default-route=true") flags.BoolVar(&o.WarnOnChannelLimit, "warn-on-channel-limit", o.WarnOnChannelLimit, "Turns on a warning if the system is going to push to a full channel. The check involves an unsafe read.") - flags.Var(&o.ApiServerMapping, "apiserver-port-mapping", "Mapping between a local port and the host:port used to reach the Kubernetes API Server") + flags.Var(&o.APIServerMapping, "apiserver-port-mapping", "Mapping between a local port and the host:port used to reach the Kubernetes API Server") flags.StringVar(&o.BindAddress, "bind-address", o.BindAddress, "Address used to listen for traffic generated on cluster network") // add feature gates flag features.DefaultMutableFeatureGate.AddFlag(flags) @@ -170,7 +170,7 @@ func (o *GrpcProxyAgentOptions) Print() { klog.V(1).Infof("WarnOnChannelLimit set to %t.\n", o.WarnOnChannelLimit) if features.DefaultMutableFeatureGate.Enabled(features.NodeToMasterTraffic) { klog.V(1).Infof("AgentBindAddress set to %s.\n", o.BindAddress) - klog.V(1).Infof("Apiserver port mapping set to %s.\n", o.ApiServerMapping.String()) + klog.V(1).Infof("Apiserver port mapping set to %s.\n", o.APIServerMapping.String()) } klog.V(1).Infof("SyncForever set to %v.\n", o.SyncForever) } @@ -223,20 +223,20 @@ func (o *GrpcProxyAgentOptions) Validate() error { if err := validateHostnameOrIP(o.BindAddress); err != nil { return fmt.Errorf("agent bind address is invalid: %v", err) } - if err := validateHostnameOrIP(o.ApiServerMapping.RemoteHost); err != nil { + if err := validateHostnameOrIP(o.APIServerMapping.RemoteHost); err != nil { return fmt.Errorf("apiserver address is invalid: %v", err) } - if o.ApiServerMapping.LocalPort > 49151 { - return fmt.Errorf("please do not try to use ephemeral port %d for the apiserver local port", o.ApiServerMapping.LocalPort) + if o.APIServerMapping.LocalPort > 49151 { + return fmt.Errorf("please do not try to use ephemeral port %d for the apiserver local port", o.APIServerMapping.LocalPort) } - if o.ApiServerMapping.LocalPort < 1024 { - return fmt.Errorf("please do not try to use reserved port %d for the apiserver local port", o.ApiServerMapping.LocalPort) + if o.APIServerMapping.LocalPort < 1024 { + return fmt.Errorf("please do not try to use reserved port %d for the apiserver local port", o.APIServerMapping.LocalPort) } - if o.ApiServerMapping.RemotePort > 49151 { - return fmt.Errorf("please do not try to use ephemeral port %d for the apiserver remote port", o.ApiServerMapping.LocalPort) + if o.APIServerMapping.RemotePort > 49151 { + return fmt.Errorf("please do not try to use ephemeral port %d for the apiserver remote port", o.APIServerMapping.LocalPort) } - if o.ApiServerMapping.RemotePort < 1 { - return fmt.Errorf("invalid port %d for the apiserver remote port", o.ApiServerMapping.RemotePort) + if o.APIServerMapping.RemotePort < 1 { + return fmt.Errorf("invalid port %d for the apiserver remote port", o.APIServerMapping.RemotePort) } return nil } @@ -294,7 +294,7 @@ func NewGrpcProxyAgentOptions() *GrpcProxyAgentOptions { ServiceAccountTokenPath: "", WarnOnChannelLimit: false, SyncForever: false, - ApiServerMapping: portMapping{LocalPort: 6443, RemoteHost: "localhost", RemotePort: 6443}, + APIServerMapping: portMapping{LocalPort: 6443, RemoteHost: "localhost", RemotePort: 6443}, BindAddress: "127.0.0.1", } return &o diff --git a/cmd/agent/app/server.go b/cmd/agent/app/server.go index 27247b941..130fab216 100644 --- a/cmd/agent/app/server.go +++ b/cmd/agent/app/server.go @@ -115,8 +115,8 @@ func (a *Agent) runControlPlaneProxy(ctx context.Context, o *options.GrpcProxyAg ClientSet: cs, ListenHost: o.BindAddress, } - klog.V(1).Infof("Exposing apiserver: %s", &o.ApiServerMapping) - return pf.Serve(ctx, agent.PortMapping(o.ApiServerMapping)) + klog.V(1).Infof("Exposing apiserver: %s", &o.APIServerMapping) + return pf.Serve(ctx, agent.PortMapping(o.APIServerMapping)) } func (a *Agent) runHealthServer(o *options.GrpcProxyAgentOptions) error { From d4eb3e09f81c562020f8c28f4511d9edada21314 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 27 Dec 2022 13:41:09 +0100 Subject: [PATCH 21/24] Fix linter warning: G102: Binds to all network interfaces (gosec) We don't need to listen on all interfaces during testing. Fixes commit 3c3912189fd7 (handle data package to agent in the same goroutine) --- pkg/server/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 0c569af98..ab44d4feb 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -265,7 +265,7 @@ func TestNodeToControlPlane(t *testing.T) { &AgentTokenAuthenticationOptions{}) // start a controlplane server - lis, err := net.Listen("tcp", ":0") + lis, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { panic(err) } From b49f476a0fee1ec3f0a066c32523310e54f1e73a Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 27 Dec 2022 13:46:18 +0100 Subject: [PATCH 22/24] Fix linter warnings in server test Ignore the linter error: G404: Use of weak random number generator (math/rand instead of crypto/rand) (gosec) Fix the linter error: receiver-naming: receiver name t should be consistent with previous receiver name f for testStream (revive) --- pkg/server/server_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index ab44d4feb..5695b71d5 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -242,12 +242,12 @@ func (f testStream) Context() context.Context { return ctx } -func (t testStream) Send(packet *client.Packet) error { +func (f testStream) Send(packet *client.Packet) error { return nil } -func (t testStream) Recv() (*client.Packet, error) { - v, ok := <-t.ch +func (f testStream) Recv() (*client.Packet, error) { + v, ok := <-f.ch if !ok { return nil, io.EOF } @@ -345,7 +345,7 @@ func packetDialReq(addr string) *client.Packet { DialRequest: &client.DialRequest{ Protocol: "tcp", Address: addr, - Random: rand.Int63(), + Random: rand.Int63(), /* #nosec G404 */ }, }, } From 11ce63c7fe3bd7b4d3f3b5b4aa74613975788603 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 27 Dec 2022 13:57:05 +0100 Subject: [PATCH 23/24] Fix linter warning about needless "= nil" assignment --- pkg/server/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 123f6a47f..4edf55cc6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -711,13 +711,13 @@ func (s *ProxyServer) Connect(stream agent.AgentService_ConnectServer) error { } klog.V(2).InfoS("Agent connected", "agentID", agentID, "serverID", s.serverID) - var nilBackend Backend = nil backend := s.addBackend(agentID, stream) + // we can't compare `backend` to nil equality because it is interface variable + var nilBackend Backend // = nil if backend == nilBackend { klog.V(2).InfoS("Unable to add backend", "agentID", agentID, "serverID", s.serverID) return fmt.Errorf("no backend added") } - // we can't compare `backend` to nil equality because it is interface variable defer s.removeBackend(agentID, stream) recvCh := make(chan *client.Packet, xfrChannelSize) From 64fd9e2a65ce1defba5d7d87bf76c9fa1703dea3 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Tue, 27 Dec 2022 14:15:15 +0100 Subject: [PATCH 24/24] Fix potential context leak and linter warning --- pkg/agent/client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/agent/client.go b/pkg/agent/client.go index 9bbed315d..7063cbb24 100644 --- a/pkg/agent/client.go +++ b/pkg/agent/client.go @@ -269,7 +269,8 @@ func newAgentClient(address, agentID, agentIdentifiers string, cs *ClientSet, op // Connect makes the grpc dial to the proxy server. It returns the serverID // it connects to. func (a *Client) Connect() (int, error) { - dialCtx, _ := context.WithTimeout(context.Background(), dialTimeout) + dialCtx, cancel := context.WithTimeout(context.Background(), dialTimeout) + defer cancel() conn, err := grpc.DialContext(dialCtx, a.address, a.opts...) if err != nil { return 0, err