metrics: add ovs client latency metrics

dist/images/install-pre-1.16.sh

@@ -1430,7 +1430,7 @@ echo ""
 echo "[Step 4] Delete pod that not in host network mode"
 for ns in $(kubectl get ns --no-headers -o  custom-columns=NAME:.metadata.name); do
   for pod in $(kubectl get pod --no-headers -n "$ns" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!="true") print $1}'); do
-    kubectl delete pod "$pod" -n "$ns"
+    kubectl delete pod "$pod" -n "$ns" --ignore-not-found
   done
 done
 

dist/images/install.sh

@@ -1695,7 +1695,7 @@ echo ""
 echo "[Step 4] Delete pod that not in host network mode"
 for ns in $(kubectl get ns --no-headers -o  custom-columns=NAME:.metadata.name); do
   for pod in $(kubectl get pod --no-headers -n "$ns" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!="true") print $1}'); do
-    kubectl delete pod "$pod" -n "$ns"
+    kubectl delete pod "$pod" -n "$ns" --ignore-not-found
   done
 done
 

pkg/ovs/adapter.go

@@ -0,0 +1,22 @@
+package ovs
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	// OVN NB metrics
+	ovsClientRequestLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "ovs_client_request_latency_milliseconds",
+			Buckets: prometheus.ExponentialBuckets(1, 2, 10),
+		},
+		[]string{"db", "method", "code"},
+	)
+)
+
+func init() {
+	registerOvsClientMetrics()
+}
+
+func registerOvsClientMetrics() {
+	prometheus.MustRegister(ovsClientRequestLatency)
+}

pkg/ovs/ovn-ic-nbctl.go

@@ -18,6 +18,18 @@ func (c Client) ovnIcNbCommand(cmdArgs ...string) (string, error) {
 		klog.Warning("ovn-ic-nbctl command error or took too long")
 		klog.Warningf("%s %s in %vms", OVNIcNbCtl, strings.Join(cmdArgs, " "), elapsed)
 	}
+	method := ""
+	for _, arg := range cmdArgs {
+		if !strings.HasPrefix(arg, "--") {
+			method = arg
+			break
+		}
+	}
+	code := "0"
+	if err != nil {
+		code = "1"
+	}
+	ovsClientRequestLatency.WithLabelValues("ovn-ic-nb", method, code).Observe(elapsed)
 	if err != nil {
 		return "", fmt.Errorf("%s, %q", raw, err)
 	}

pkg/ovs/ovn-nbctl.go

@@ -24,6 +24,18 @@ func (c Client) ovnNbCommand(cmdArgs ...string) (string, error) {
 		klog.Warning("ovn-nbctl command error or took too long")
 		klog.Warningf("%s %s in %vms", OvnNbCtl, strings.Join(cmdArgs, " "), elapsed)
 	}
+	method := ""
+	for _, arg := range cmdArgs {
+		if !strings.HasPrefix(arg, "--") {
+			method = arg
+			break
+		}
+	}
+	code := "0"
+	if err != nil {
+		code = "1"
+	}
+	ovsClientRequestLatency.WithLabelValues("ovn-nb", method, code).Observe(elapsed)
 	if err != nil {
 		return "", fmt.Errorf("%s, %q", raw, err)
 	}

pkg/ovs/ovn-sbctl.go

@@ -19,6 +19,18 @@ func (c Client) ovnSbCommand(cmdArgs ...string) (string, error) {
 		klog.Warning("ovn-sbctl command error or took too long")
 		klog.Warningf("%s %s in %vms", OvnSbCtl, strings.Join(cmdArgs, " "), elapsed)
 	}
+	method := ""
+	for _, arg := range cmdArgs {
+		if !strings.HasPrefix(arg, "--") {
+			method = arg
+			break
+		}
+	}
+	code := "0"
+	if err != nil {
+		code = "1"
+	}
+	ovsClientRequestLatency.WithLabelValues("ovn-sb", method, code).Observe(elapsed)
 	if err != nil {
 		return "", fmt.Errorf("%s, %q", raw, err)
 	}

pkg/ovs/ovs-vsctl.go

@@ -22,6 +22,18 @@ func Exec(args ...string) (string, error) {
 		klog.Warning("ovs-vsctl command error or took too long")
 		klog.Warningf("ovs-vsctl %s in %vms", strings.Join(args, " "), elapsed)
 	}
+	method := ""
+	for _, arg := range args {
+		if !strings.HasPrefix(arg, "--") {
+			method = arg
+			break
+		}
+	}
+	code := "0"
+	if err != nil {
+		code = "1"
+	}
+	ovsClientRequestLatency.WithLabelValues("ovsdb", method, code).Observe(elapsed)
 	if err != nil {
 		return "", fmt.Errorf("failed to run 'ovs-vsctl %s': %v\n  %q", strings.Join(args, " "), err, output)
 	}