feat: pinger add apiserver check metrics

kubeovn · Oct 15, 2019 · ef285b2 · ef285b2
1 parent 02f4adc
commit ef285b2
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 2 deletions.
diff --git a/pkg/controller/config.go b/pkg/controller/config.go
@@ -4,6 +4,7 @@ import (
 	"flag"
 	"github.com/alauda/kube-ovn/pkg/util"
 	"os"
+	"time"
 
 	clientset "github.com/alauda/kube-ovn/pkg/client/clientset/versioned"
 	"github.com/spf13/pflag"
@@ -155,6 +156,7 @@ func (config *Configuration) initKubeClient() error {
 			return err
 		}
 	}
+	cfg.Timeout = 15 * time.Second
 	kubeClient, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		klog.Errorf("init kubernetes client failed %v", err)

diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go
@@ -9,6 +9,7 @@ import (
 	"os/exec"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/alauda/kube-ovn/pkg/util"
 	"github.com/sirupsen/logrus"
@@ -145,6 +146,7 @@ func (config *Configuration) initKubeClient() error {
 			return err
 		}
 	}
+	cfg.Timeout = 15 * time.Second
 
 	kubeClient, err := kubernetes.NewForConfig(cfg)
 	if err != nil {

diff --git a/pkg/pinger/config.go b/pkg/pinger/config.go
@@ -8,6 +8,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog"
 	"os"
+	"time"
 )
 
 type Configuration struct {
@@ -88,7 +89,7 @@ func (config *Configuration) initKubeClient() error {
 			return err
 		}
 	}
-
+	cfg.Timeout = 15 * time.Second
 	kubeClient, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		klog.Errorf("init kubernetes client failed %v", err)

diff --git a/pkg/pinger/metrics.go b/pkg/pinger/metrics.go
@@ -35,6 +35,31 @@ var (
 		[]string{
 			"nodeName",
 		})
+	apiserverHealthyGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "pinger_apiserver_healthy",
+			Help: "if the apiserver request is healthy on this node",
+		},
+		[]string{
+			"nodeName",
+		})
+	apiserverUnhealthyGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "pinger_apiserver_unhealthy",
+			Help: "if the apiserver request is unhealthy on this node",
+		},
+		[]string{
+			"nodeName",
+		})
+	apiserverRequestLatencyHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "pinger_apiserver_latency_ms",
+			Help:    "the latency ms histogram the node request apiserver",
+			Buckets: []float64{.5, 1, 2, 5, 10, 30},
+		},
+		[]string{
+			"nodeName",
+		})
 	dnsHealthyGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "pinger_dns_healthy",
@@ -54,7 +79,7 @@ var (
 	dnsRequestLatencyHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "pinger_dns_latency_ms",
-			Help:    "the latency ms histogram the node request dsn",
+			Help:    "the latency ms histogram the node request dns",
 			Buckets: []float64{.5, 1, 2, 5, 10, 30},
 		},
 		[]string{
@@ -117,6 +142,9 @@ func init() {
 	prometheus.MustRegister(ovsDownGauge)
 	prometheus.MustRegister(ovnControllerUpGauge)
 	prometheus.MustRegister(ovnControllerDownGauge)
+	prometheus.MustRegister(apiserverHealthyGauge)
+	prometheus.MustRegister(apiserverUnhealthyGauge)
+	prometheus.MustRegister(apiserverRequestLatencyHistogram)
 	prometheus.MustRegister(dnsHealthyGauge)
 	prometheus.MustRegister(dnsUnhealthyGauge)
 	prometheus.MustRegister(dnsRequestLatencyHistogram)
@@ -146,6 +174,17 @@ func SetOvnControllerDownMetrics(nodeName string) {
 	ovnControllerDownGauge.WithLabelValues(nodeName).Set(1)
 }
 
+func SetApiserverHealthyMetrics(nodeName string, latency float64) {
+	apiserverHealthyGauge.WithLabelValues(nodeName).Set(1)
+	apiserverRequestLatencyHistogram.WithLabelValues(nodeName).Observe(latency)
+	apiserverUnhealthyGauge.WithLabelValues(nodeName).Set(0)
+}
+
+func SetApiserverUnhealthyMetrics(nodeName string) {
+	apiserverHealthyGauge.WithLabelValues(nodeName).Set(0)
+	apiserverUnhealthyGauge.WithLabelValues(nodeName).Set(1)
+}
+
 func SetDnsHealthyMetrics(nodeName string, latency float64) {
 	dnsHealthyGauge.WithLabelValues(nodeName).Set(1)
 	dnsRequestLatencyHistogram.WithLabelValues(nodeName).Observe(latency)

diff --git a/pkg/pinger/ping.go b/pkg/pinger/ping.go
@@ -17,6 +17,7 @@ func StartPinger(config *Configuration) {
 	for {
 		checkOvs(config)
 		checkOvnController(config)
+		checkApiServer(config)
 		ping(config)
 		if config.Mode != "server" {
 			break
@@ -151,3 +152,18 @@ func checkOvnController(config *Configuration) {
 	klog.Infof("ovn_controller is up")
 	SetOvnControllerUpMetrics(config.NodeName)
 }
+
+func checkApiServer(config *Configuration) {
+	klog.Infof("start to check apiserver connectivity")
+	t1 := time.Now()
+	_, err := config.KubeClient.Discovery().ServerVersion()
+	elpased := time.Since(t1)
+	if err != nil {
+		klog.Errorf("failed to connect to apiserver: %v", err)
+		SetApiserverUnhealthyMetrics(config.NodeName)
+		return
+	}
+	klog.Infof("connect to apiserver success in %.2fms", float64(elpased)/float64(time.Millisecond))
+	SetApiserverHealthyMetrics(config.NodeName, float64(elpased)/float64(time.Millisecond))
+	return
+}
diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go
@@ -2,6 +2,7 @@ package webhook
 
 import (
 	"context"
+	"time"
 
 	clientset "github.com/alauda/kube-ovn/pkg/client/clientset/versioned"
 	"github.com/alauda/kube-ovn/pkg/ovs"
@@ -39,6 +40,7 @@ func NewValidatingHook(c cache.Cache, opt *WebhookOptions) (*ValidatingHook, err
 		klog.Errorf("use in cluster config failed %v", err)
 		return nil, err
 	}
+	cfg.Timeout = 15 * time.Second
 	kubeClient, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		klog.Errorf("init kubernetes client failed %v", err)