From c98ae6bf27177a241ba28e33c573a75c2e663433 Mon Sep 17 00:00:00 2001 From: Paul Miller Date: Fri, 18 Mar 2022 19:35:19 +0000 Subject: [PATCH 1/2] combine metrics and healthz --- cns/service/main.go | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cns/service/main.go b/cns/service/main.go index 645b8fed8a..64ce7bf52a 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -46,6 +46,7 @@ import ( "github.com/Azure/azure-container-networking/store" "github.com/avast/retry-go/v3" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus/promhttp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" @@ -53,6 +54,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) const ( @@ -991,10 +994,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn }) manager, err := ctrl.NewManager(kubeConfig, ctrl.Options{ - Scheme: nodenetworkconfig.Scheme, - MetricsBindAddress: cnsconfig.MetricsBindAddress, - Namespace: "kube-system", // TODO(rbtr): namespace should be in the cns config - NewCache: nodeScopedCache, + Scheme: nodenetworkconfig.Scheme, + Namespace: "kube-system", // TODO(rbtr): namespace should be in the cns config + NewCache: nodeScopedCache, }) if err != nil { return errors.Wrap(err, "failed to create manager") @@ -1013,6 +1015,28 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn return errors.Wrapf(err, "failed to setup reconciler with manager") } + mux := http.NewServeMux() + + /*pprof endpoints + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + */ + + //to reduce + mux.Handle("/metrics", promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{ + ErrorHandling: promhttp.HTTPErrorOnError, + })) + healthzhandler := healthz.Handler{} + healthzhandler.Checks["ping"] = healthz.Ping + mux.Handle("/heathz", &healthzhandler) + + go func() { + http.ListenAndServe(cnsconfig.MetricsBindAddress, mux) + }() + // Start the Manager which starts the reconcile loop. // The Reconciler will send an initial NodeNetworkConfig update to the PoolMonitor, starting the // Monitor's internal loop. @@ -1057,6 +1081,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } } }() + logger.Printf("initialized and started SyncHostNCVersion loop") return nil From 15826330abe8d15c8f7b588f2006afbc39cb0920 Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Mon, 4 Apr 2022 18:06:57 +0000 Subject: [PATCH 2/2] use root mux and bind pprof routes on debug config opt Signed-off-by: Evan Baker --- cns/configuration/configuration.go | 5 +--- cns/configuration/configuration_test.go | 5 +--- cns/service/main.go | 34 ++++++++++++------------- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index b1414a2fc7..b4547d82f8 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -22,7 +22,7 @@ type CNSConfig struct { ChannelMode string InitializeFromCNI bool ManagedSettings ManagedSettings - MetricsBindAddress string + Debug bool SyncHostNCTimeoutMs int SyncHostNCVersionIntervalMs int TLSCertificatePath string @@ -150,9 +150,6 @@ func SetCNSConfigDefaults(config *CNSConfig) { if config.ChannelMode == "" { config.ChannelMode = cns.Direct } - if config.MetricsBindAddress == "" { - config.MetricsBindAddress = ":9090" - } if config.SyncHostNCVersionIntervalMs == 0 { config.SyncHostNCVersionIntervalMs = 1000 //nolint:gomnd // default times } diff --git a/cns/configuration/configuration_test.go b/cns/configuration/configuration_test.go index eee4c61b99..c7e79f80a8 100644 --- a/cns/configuration/configuration_test.go +++ b/cns/configuration/configuration_test.go @@ -61,7 +61,6 @@ func TestReadConfigFromFile(t *testing.T) { NodeID: "abc", NodeSyncIntervalInSeconds: 30, }, - MetricsBindAddress: ":9091", SyncHostNCTimeoutMs: 5, SyncHostNCVersionIntervalMs: 5, TLSCertificatePath: "/test", @@ -191,7 +190,7 @@ func TestSetCNSConfigDefaults(t *testing.T) { ManagedSettings: ManagedSettings{ NodeSyncIntervalInSeconds: 30, }, - MetricsBindAddress: ":9090", + Debug: false, SyncHostNCTimeoutMs: 500, SyncHostNCVersionIntervalMs: 1000, TelemetrySettings: TelemetrySettings{ @@ -210,7 +209,6 @@ func TestSetCNSConfigDefaults(t *testing.T) { ManagedSettings: ManagedSettings{ NodeSyncIntervalInSeconds: 1, }, - MetricsBindAddress: ":9091", SyncHostNCTimeoutMs: 5, SyncHostNCVersionIntervalMs: 1, TelemetrySettings: TelemetrySettings{ @@ -226,7 +224,6 @@ func TestSetCNSConfigDefaults(t *testing.T) { ManagedSettings: ManagedSettings{ NodeSyncIntervalInSeconds: 1, }, - MetricsBindAddress: ":9091", SyncHostNCTimeoutMs: 5, SyncHostNCVersionIntervalMs: 1, TelemetrySettings: TelemetrySettings{ diff --git a/cns/service/main.go b/cns/service/main.go index 64ce7bf52a..5abaf7069f 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/http/pprof" "os" "os/signal" "runtime" @@ -337,8 +338,8 @@ func sendRegisterNodeRequest( httpc *http.Client, httpRestService cns.HTTPService, nodeRegisterRequest cns.NodeRegisterRequest, - registerURL string) error { - + registerURL string, +) error { var body bytes.Buffer err := json.NewEncoder(&body).Encode(nodeRegisterRequest) if err != nil { @@ -1015,27 +1016,26 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn return errors.Wrapf(err, "failed to setup reconciler with manager") } - mux := http.NewServeMux() + // adding some routes to the root serve mux + mux := httpRestServiceImplementation.Listener.GetMux() - /*pprof endpoints - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - */ + if cnsconfig.Debug { + // add pprof endpoints + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + } - //to reduce + // add metrics endpoints mux.Handle("/metrics", promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{ ErrorHandling: promhttp.HTTPErrorOnError, })) - healthzhandler := healthz.Handler{} - healthzhandler.Checks["ping"] = healthz.Ping - mux.Handle("/heathz", &healthzhandler) - go func() { - http.ListenAndServe(cnsconfig.MetricsBindAddress, mux) - }() + // add healthz endpoints + healthzhandler := healthz.Handler{} + mux.Handle("/healthz", http.StripPrefix("/healthz", &healthzhandler)) // Start the Manager which starts the reconcile loop. // The Reconciler will send an initial NodeNetworkConfig update to the PoolMonitor, starting the