local ip bind to service (#2195)

* local ip bind to service

* enable-bind-local-ip set true by default

* add configure params for bind local ip

* ovn-is-leader.sh should listen local ip

cmd/controller/controller.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -19,6 +20,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/controller"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
 	"github.com/kubeovn/kube-ovn/pkg/util"
@@ -64,10 +66,23 @@ func CmdMain() {
 			mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 		}
 
+		addr := "0.0.0.0"
+		if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
+			podIpsEnv := os.Getenv("POD_IPS")
+			podIps := strings.Split(podIpsEnv, ",")
+			// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
+			// so not support bind local ip when in dual mode
+			if len(podIps) == 1 {
+				addr = podIps[0]
+				if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
+					addr = fmt.Sprintf("[%s]", podIps[0])
+				}
+			}
+		}
 		// conform to Gosec G114
 		// https://github.com/securego/gosec#available-rules
 		server := &http.Server{
-			Addr:              fmt.Sprintf("0.0.0.0:%d", config.PprofPort),
+			Addr:              fmt.Sprintf("%s:%d", addr, config.PprofPort),
 			ReadHeaderTimeout: 3 * time.Second,
 			Handler:           mux,
 		}

cmd/controller_health_check/controller_health_check.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 )
@@ -23,7 +24,22 @@ func CmdMain() {
 	if err := ovs.CheckAlive(); err != nil {
 		os.Exit(1)
 	}
-	conn, err := net.DialTimeout("tcp", "127.0.0.1:10660", 3*time.Second)
+
+	addr := "127.0.0.1:10660"
+	if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
+		podIpsEnv := os.Getenv("POD_IPS")
+		podIps := strings.Split(podIpsEnv, ",")
+		// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
+		// so not support bind local ip when in dual mode
+		if len(podIps) == 1 {
+			addr = fmt.Sprintf("%s:10660", podIps[0])
+			if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
+				addr = fmt.Sprintf("[%s]:10660", podIps[0])
+			}
+		}
+	}
+
+	conn, err := net.DialTimeout("tcp", addr, 3*time.Second)
 	if err != nil {
 		util.LogFatalAndExit(err, "failed to probe the socket")
 	}

cmd/daemon/cniserver.go

@@ -18,6 +18,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/sample-controller/pkg/signals"
 
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
 	"github.com/kubeovn/kube-ovn/pkg/daemon"
 	"github.com/kubeovn/kube-ovn/pkg/util"
@@ -95,10 +96,23 @@ func CmdMain() {
 		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 	}
 
+	addr := "0.0.0.0"
+	if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
+		podIpsEnv := os.Getenv("POD_IPS")
+		podIps := strings.Split(podIpsEnv, ",")
+		// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
+		// so not support bind local ip when in dual mode
+		if len(podIps) == 1 {
+			addr = podIps[0]
+			if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
+				addr = fmt.Sprintf("[%s]", podIps[0])
+			}
+		}
+	}
 	// conform to Gosec G114
 	// https://github.com/securego/gosec#available-rules
 	server := &http.Server{
-		Addr:              fmt.Sprintf("0.0.0.0:%d", config.PprofPort),
+		Addr:              fmt.Sprintf("%s:%d", addr, config.PprofPort),
 		ReadHeaderTimeout: 3 * time.Second,
 		Handler:           mux,
 	}

cmd/ovn_monitor/ovn_monitor.go

@@ -1,12 +1,16 @@
 package ovn_monitor
 
 import (
+	"fmt"
 	"net/http"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/klog/v2"
 
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	ovn "github.com/kubeovn/kube-ovn/pkg/ovnmonitor"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
@@ -33,8 +37,23 @@ func CmdMain() {
 
 	// conform to Gosec G114
 	// https://github.com/securego/gosec#available-rules
+
+	addr := config.ListenAddress
+	if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
+		podIpsEnv := os.Getenv("POD_IPS")
+		podIps := strings.Split(podIpsEnv, ",")
+		// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
+		// so not support bind local ip when in dual mode
+		if len(podIps) == 1 {
+			addr = fmt.Sprintf("%s:10661", podIps[0])
+			if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
+				addr = fmt.Sprintf("[%s]:10661", podIps[0])
+			}
+		}
+	}
+
 	server := &http.Server{
-		Addr:              config.ListenAddress,
+		Addr:              addr,
 		ReadHeaderTimeout: 3 * time.Second,
 	}
 	util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", config.ListenAddress)

dist/images/install.sh

@@ -27,6 +27,7 @@ IFACE=${IFACE:-}
 # Specifies the name of the dpdk tunnel iface.
 # Note that the dpdk tunnel iface and tunnel ip cidr should be diffierent with Kubernetes api cidr,otherwise the route will be a problem.
 DPDK_TUNNEL_IFACE=${DPDK_TUNNEL_IFACE:-br-phy}
+ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-true}
 
 CNI_CONF_DIR="/etc/cni/net.d"
 CNI_BIN_DIR="/opt/cni/bin"
@@ -2023,6 +2024,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "$ENABLE_BIND_LOCAL_IP"
           resources:
             requests:
               cpu: 300m
@@ -2517,6 +2524,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "$ENABLE_BIND_LOCAL_IP"
           resources:
             requests:
               cpu: 300m
@@ -3016,6 +3029,12 @@ spec:
                   fieldPath: spec.nodeName
             - name: OVN_DB_IPS
               value: $addresses
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "$ENABLE_BIND_LOCAL_IP"
           volumeMounts:
             - mountPath: /etc/localtime
               name: localtime
@@ -3140,6 +3159,12 @@ spec:
             value: $MODULES
           - name: RPMS
             value: $RPMS
+          - name: POD_IPS
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIPs
+          - name: ENABLE_BIND_LOCAL_IP
+            value: "$ENABLE_BIND_LOCAL_IP"
         volumeMounts:
           - name: host-modules
             mountPath: /lib/modules
@@ -3419,6 +3444,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "$ENABLE_BIND_LOCAL_IP"
           resources:
             requests:
               cpu: 200m

dist/images/ovn-is-leader.sh

@@ -8,12 +8,24 @@ ovn-ctl status_northd
 ovn-ctl status_ovnnb
 ovn-ctl status_ovnsb
 
+BIND_LOCAL_ADDR=127.0.0.1
+if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
+  POD_IPS_LIST=(${POD_IPS//,/ })
+  if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then
+    if [[ $POD_IP =~ .*:.* ]]; then
+      BIND_LOCAL_ADDR=[${POD_IP}] #ipv6
+    else
+      BIND_LOCAL_ADDR=${POD_IP} #ipv4
+    fi
+  fi
+fi
+
 # For data consistency, only store leader address in endpoint
 # Store ovn-nb leader to svc kube-system/ovn-nb
 if [[ "$ENABLE_SSL" == "false" ]]; then
-  nb_leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+  nb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 else
-  nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+  nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 fi
 
 if [[ $nb_leader =~ "true" ]]
@@ -34,9 +46,9 @@ fi
 
 # Store ovn-sb leader to svc kube-system/ovn-sb
 if [[ "$ENABLE_SSL" == "false" ]]; then
-  sb_leader=$(ovsdb-client query tcp:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+  sb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 else
-  sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+  sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 fi
 
 if [[ $sb_leader =~ "true" ]]
@@ -51,9 +63,9 @@ then
     if [ "$northd_leader" == "" ]; then
        # no available northd leader try to release the lock
        if [[ "$ENABLE_SSL" == "false" ]]; then
-         ovsdb-client -v -t 1 steal tcp:127.0.0.1:6642  ovn_northd
+         ovsdb-client -v -t 1 steal tcp:$BIND_LOCAL_ADDR:6642  ovn_northd
        else
-         ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:127.0.0.1:6642  ovn_northd
+         ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:$BIND_LOCAL_ADDR:6642  ovn_northd
        fi
      fi
    fi

dist/images/start-db.sh

@@ -25,6 +25,14 @@ DB_NB_PORT=${DB_NB_PORT:-6641}
 DB_SB_ADDR=${DB_SB_ADDR:-::}
 DB_SB_PORT=${DB_SB_PORT:-6642}
 ENABLE_SSL=${ENABLE_SSL:-false}
+ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-false}
+BIND_LOCAL_ADDR=[::]
+if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
+  POD_IPS_LIST=(${POD_IPS//,/ })
+  if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then
+    BIND_LOCAL_ADDR="[${POD_IP}]"
+  fi
+fi
 
 . /usr/share/openvswitch/scripts/ovs-lib || exit 1
 
@@ -177,8 +185,8 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
                 --db-sb-create-insecure-remote=yes \
                 --db-nb-cluster-local-addr="[${POD_IP}]" \
                 --db-sb-cluster-local-addr="[${POD_IP}]" \
-                --db-nb-addr=[::] \
-                --db-sb-addr=[::] \
+                --db-nb-addr=$BIND_LOCAL_ADDR \
+                --db-sb-addr=$BIND_LOCAL_ADDR \
                 --ovn-northd-nb-db="$(gen_conn_str 6641)" \
                 --ovn-northd-sb-db="$(gen_conn_str 6642)" \
                 start_northd
@@ -222,8 +230,8 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
                 --db-sb-cluster-local-addr="[${POD_IP}]" \
                 --db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
                 --db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
-                --db-nb-addr=[::] \
-                --db-sb-addr=[::] \
+                --db-nb-addr=$BIND_LOCAL_ADDR \
+                --db-sb-addr=$BIND_LOCAL_ADDR \
                 --ovn-northd-nb-db="$(gen_conn_str 6641)" \
                 --ovn-northd-sb-db="$(gen_conn_str 6642)" \
                 start_northd
@@ -277,16 +285,16 @@ else
                 --ovn-northd-ssl-ca-cert=/var/run/tls/cacert \
                 --db-nb-cluster-local-addr="[${POD_IP}]" \
                 --db-sb-cluster-local-addr="[${POD_IP}]" \
-                --db-nb-addr=[::] \
-                --db-sb-addr=[::] \
+                --db-nb-addr=$BIND_LOCAL_ADDR \
+                --db-sb-addr=$BIND_LOCAL_ADDR \
                 --ovn-northd-nb-db="$(gen_conn_str 6641)" \
                 --ovn-northd-sb-db="$(gen_conn_str 6642)" \
                 start_northd
-            ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":[::]
+            ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":$BIND_LOCAL_ADDR
             ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
             ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:use_logical_dp_groups=true
 
-            ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":[::]
+            ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":$BIND_LOCAL_ADDR
             ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
         else
             # get leader if cluster exists
@@ -328,8 +336,8 @@ else
                 --db-sb-cluster-local-addr="[${POD_IP}]" \
                 --db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
                 --db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
-                --db-nb-addr=[::] \
-                --db-sb-addr=[::] \
+                --db-nb-addr=$BIND_LOCAL_ADDR \
+                --db-sb-addr=$BIND_LOCAL_ADDR \
                 --ovn-northd-nb-db="$(gen_conn_str 6641)" \
                 --ovn-northd-sb-db="$(gen_conn_str 6642)" \
                 start_northd

kubeovn-helm/templates/central-deploy.yaml

@@ -65,6 +65,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "{{- .Values.func.ENABLE_BIND_LOCAL_IP }}"
           resources:
             requests:
               cpu: 300m

kubeovn-helm/templates/controller-deploy.yaml

@@ -115,6 +115,12 @@ spec:
                   fieldPath: spec.nodeName
             - name: OVN_DB_IPS
               value: "{{ .Values.MASTER_NODES }}"
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "{{- .Values.func.ENABLE_BIND_LOCAL_IP }}"
           volumeMounts:
             - mountPath: /etc/localtime
               name: localtime

kubeovn-helm/templates/monitor-deploy.yaml

@@ -53,6 +53,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "{{- .Values.func.ENABLE_BIND_LOCAL_IP }}"
           resources:
             requests:
               cpu: 200m

kubeovn-helm/templates/ovncni-ds.yaml

@@ -90,6 +90,12 @@ spec:
             value: "{{- .Values.performance.MODULES }}"
           - name: RPMS
             value: "{{- .Values.performance.RPMS }}"
+          - name: POD_IPS
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIPs
+          - name: ENABLE_BIND_LOCAL_IP
+            value: "{{- .Values.func.ENABLE_BIND_LOCAL_IP }}"
         volumeMounts:
           - name: host-modules
             mountPath: /lib/modules