Skip to content

move metrics to service port, add healthz, pprof #1318

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 5, 2022
Merged

move metrics to service port, add healthz, pprof #1318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c98ae6b
combine metrics and healthz
paulgmiller Mar 18, 2022
1582633
use root mux and bind pprof routes on debug config opt
rbtr Apr 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions cns/configuration/configuration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type CNSConfig struct {
ChannelMode string
InitializeFromCNI bool
ManagedSettings ManagedSettings
MetricsBindAddress string
Debug bool
SyncHostNCTimeoutMs int
SyncHostNCVersionIntervalMs int
TLSCertificatePath string
Expand Down Expand Up @@ -150,9 +150,6 @@ func SetCNSConfigDefaults(config *CNSConfig) {
if config.ChannelMode == "" {
config.ChannelMode = cns.Direct
}
if config.MetricsBindAddress == "" {
config.MetricsBindAddress = ":9090"
}
if config.SyncHostNCVersionIntervalMs == 0 {
config.SyncHostNCVersionIntervalMs = 1000 //nolint:gomnd // default times
}
Expand Down
5 changes: 1 addition & 4 deletions cns/configuration/configuration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ func TestReadConfigFromFile(t *testing.T) {
NodeID: "abc",
NodeSyncIntervalInSeconds: 30,
},
MetricsBindAddress: ":9091",
SyncHostNCTimeoutMs: 5,
SyncHostNCVersionIntervalMs: 5,
TLSCertificatePath: "/test",
Expand Down Expand Up @@ -191,7 +190,7 @@ func TestSetCNSConfigDefaults(t *testing.T) {
ManagedSettings: ManagedSettings{
NodeSyncIntervalInSeconds: 30,
},
MetricsBindAddress: ":9090",
Debug: false,
SyncHostNCTimeoutMs: 500,
SyncHostNCVersionIntervalMs: 1000,
TelemetrySettings: TelemetrySettings{
Expand All @@ -210,7 +209,6 @@ func TestSetCNSConfigDefaults(t *testing.T) {
ManagedSettings: ManagedSettings{
NodeSyncIntervalInSeconds: 1,
},
MetricsBindAddress: ":9091",
SyncHostNCTimeoutMs: 5,
SyncHostNCVersionIntervalMs: 1,
TelemetrySettings: TelemetrySettings{
Expand All @@ -226,7 +224,6 @@ func TestSetCNSConfigDefaults(t *testing.T) {
ManagedSettings: ManagedSettings{
NodeSyncIntervalInSeconds: 1,
},
MetricsBindAddress: ":9091",
SyncHostNCTimeoutMs: 5,
SyncHostNCVersionIntervalMs: 1,
TelemetrySettings: TelemetrySettings{
Expand Down
37 changes: 31 additions & 6 deletions cns/service/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"net/http/pprof"
"os"
"os/signal"
"runtime"
Expand Down Expand Up @@ -46,13 +47,16 @@ import (
"github.com/Azure/azure-container-networking/store"
"github.com/avast/retry-go/v3"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)

const (
Expand Down Expand Up @@ -334,8 +338,8 @@ func sendRegisterNodeRequest(
httpc *http.Client,
httpRestService cns.HTTPService,
nodeRegisterRequest cns.NodeRegisterRequest,
registerURL string) error {

registerURL string,
) error {
var body bytes.Buffer
err := json.NewEncoder(&body).Encode(nodeRegisterRequest)
if err != nil {
Expand Down Expand Up @@ -991,10 +995,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
})

manager, err := ctrl.NewManager(kubeConfig, ctrl.Options{
Scheme: nodenetworkconfig.Scheme,
MetricsBindAddress: cnsconfig.MetricsBindAddress,
Namespace: "kube-system", // TODO(rbtr): namespace should be in the cns config
NewCache: nodeScopedCache,
Scheme: nodenetworkconfig.Scheme,
Namespace: "kube-system", // TODO(rbtr): namespace should be in the cns config
NewCache: nodeScopedCache,
})
if err != nil {
return errors.Wrap(err, "failed to create manager")
Expand All @@ -1013,6 +1016,27 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
return errors.Wrapf(err, "failed to setup reconciler with manager")
}

// adding some routes to the root serve mux
mux := httpRestServiceImplementation.Listener.GetMux()
Copy link
Member

@paulgmiller paulgmiller Apr 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we have to remember to update the metrics annotation port when we bring this in. Weee non atomic changes.


if cnsconfig.Debug {
// add pprof endpoints
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
}

// add metrics endpoints
mux.Handle("/metrics", promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{
ErrorHandling: promhttp.HTTPErrorOnError,
}))

// add healthz endpoints
healthzhandler := healthz.Handler{}
Copy link
Member

@paulgmiller paulgmiller Apr 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no idea if this little heathz framework from controller-runtime is good or not. Feel freee to toss.

mux.Handle("/healthz", http.StripPrefix("/healthz", &healthzhandler))

// Start the Manager which starts the reconcile loop.
// The Reconciler will send an initial NodeNetworkConfig update to the PoolMonitor, starting the
// Monitor's internal loop.
Expand Down Expand Up @@ -1057,6 +1081,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
}
}
}()

logger.Printf("initialized and started SyncHostNCVersion loop")

return nil
Expand Down