metrics: expose cni operation metrics

cmd/daemon/cniserver.go

@@ -8,6 +8,7 @@ import (
 
 	kubeovninformer "github.com/alauda/kube-ovn/pkg/client/informers/externalversions"
 	"github.com/alauda/kube-ovn/pkg/daemon"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog"
 	"k8s.io/sample-controller/pkg/signals"
@@ -46,7 +47,8 @@ func main() {
 	kubeovnInformerFactory.Start(stopCh)
 	go ctl.Run(stopCh)
 	go func() {
-		klog.Fatal(http.ListenAndServe(fmt.Sprintf("localhost:%d", config.PprofPort), nil))
+		http.Handle("/metrics", promhttp.Handler())
+		klog.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", config.PprofPort), nil))
 	}()
 	daemon.RunServer(config)
 }

dist/monitoring/cni-monitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kube-ovn-cni
+  namespace: monitoring
+spec:
+  endpoints:
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      interval: 15s
+      port: metrics
+  namespaceSelector:
+    matchNames:
+      - kube-system
+  selector:
+    matchLabels:
+      app: kube-ovn-cni

pkg/daemon/metrics.go

@@ -0,0 +1,21 @@
+package daemon
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	nodeName              = ""
+	cniOperationHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "cni_op_latency_second",
+			Help:    "the latency second for cni operations",
+			Buckets: []float64{.1, .25, .5, 1, 2, 4, 8, 16, 32, 64, 128, 256},
+		}, []string{
+			"node_name",
+			"method",
+			"status_code",
+		})
+)
+
+func init() {
+	prometheus.MustRegister(cniOperationHistogram)
+}

pkg/daemon/server.go

@@ -17,6 +17,7 @@ var responseLogString = "[%s] Outgoing response %s %s with %d status code in %vm
 
 // RunServer runs the cniserver
 func RunServer(config *Configuration) {
+	nodeName = config.NodeName
 	csh := createCniServerHandler(config)
 	server := http.Server{
 		Handler: createHandler(csh),
@@ -62,6 +63,10 @@ func requestAndResponseLogger(request *restful.Request, response *restful.Respon
 	start := time.Now()
 	chain.ProcessFilter(request, response)
 	elapsed := float64((time.Since(start)) / time.Millisecond)
+	cniOperationHistogram.WithLabelValues(
+		nodeName,
+		getRequestURI(request),
+		fmt.Sprintf("%d", response.StatusCode())).Observe(elapsed / 1000)
 	klog.Infof(formatResponseLog(response, request, elapsed))
 }
 

yamls/kube-ovn.yaml

@@ -302,3 +302,17 @@ spec:
   ports:
     - port: 10660
       name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+  labels:
+    app: kube-ovn-cni
+spec:
+  selector:
+    app: kube-ovn-cni
+  ports:
+    - port: 10665
+      name: metrics