Skip to content

Commit

Permalink
bind local ip release 1.10 (#2198)
Browse files Browse the repository at this point in the history
  • Loading branch information
@changluyi
changluyi committed Dec 29, 2022
1 parent 2cad035 commit 7dba66c
Show file tree
Hide file tree
Showing 14 changed files with 227 additions and 24 deletions.
24 changes: 23 additions & 1 deletion cmd/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net/http"
"net/http/pprof"
"os"
"strings"
"time"

"github.com/prometheus/client_golang/prometheus/promhttp"
Expand All @@ -14,6 +15,7 @@ import (
"k8s.io/klog/v2"
"k8s.io/sample-controller/pkg/signals"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/controller"
"github.com/kubeovn/kube-ovn/pkg/ovs"
"github.com/kubeovn/kube-ovn/pkg/util"
Expand Down Expand Up @@ -49,7 +51,27 @@ func CmdMain() {
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
}
klog.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", config.PprofPort), mux))
addr := "0.0.0.0"
if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
podIpsEnv := os.Getenv("POD_IPS")
podIps := strings.Split(podIpsEnv, ",")
// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
// so not support bind local ip when in dual mode
if len(podIps) == 1 {
addr = podIps[0]
if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
addr = fmt.Sprintf("[%s]", podIps[0])
}
}
}
// conform to Gosec G114
// https://github.com/securego/gosec#available-rules
server := &http.Server{
Addr: fmt.Sprintf("%s:%d", addr, config.PprofPort),
ReadHeaderTimeout: 3 * time.Second,
Handler: mux,
}
util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
}()

ctl := controller.NewController(config)
Expand Down
18 changes: 17 additions & 1 deletion cmd/controller_health_check/controller_health_check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strings"
"time"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/ovs"
"github.com/kubeovn/kube-ovn/pkg/util"
)
Expand All @@ -23,7 +24,22 @@ func CmdMain() {
if err := ovs.CheckAlive(); err != nil {
os.Exit(1)
}
conn, err := net.DialTimeout("tcp", "127.0.0.1:10660", 3*time.Second)

addr := "127.0.0.1:10660"
if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
podIpsEnv := os.Getenv("POD_IPS")
podIps := strings.Split(podIpsEnv, ",")
// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
// so not support bind local ip when in dual mode
if len(podIps) == 1 {
addr = fmt.Sprintf("%s:10660", podIps[0])
if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
addr = fmt.Sprintf("[%s]:10660", podIps[0])
}
}
}

conn, err := net.DialTimeout("tcp", addr, 3*time.Second)
if err != nil {
util.LogFatalAndExit(err, "failed to probe the socket")
}
Expand Down
24 changes: 23 additions & 1 deletion cmd/daemon/cniserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"k8s.io/klog/v2"
"k8s.io/sample-controller/pkg/signals"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
"github.com/kubeovn/kube-ovn/pkg/daemon"
"github.com/kubeovn/kube-ovn/pkg/util"
Expand Down Expand Up @@ -94,7 +95,28 @@ func CmdMain() {
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
}
klog.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", config.PprofPort), mux))

addr := "0.0.0.0"
if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
podIpsEnv := os.Getenv("POD_IPS")
podIps := strings.Split(podIpsEnv, ",")
// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
// so not support bind local ip when in dual mode
if len(podIps) == 1 {
addr = podIps[0]
if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
addr = fmt.Sprintf("[%s]", podIps[0])
}
}
}
// conform to Gosec G114
// https://github.com/securego/gosec#available-rules
server := &http.Server{
Addr: fmt.Sprintf("%s:%d", addr, config.PprofPort),
ReadHeaderTimeout: 3 * time.Second,
Handler: mux,
}
util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and serve on %s", server.Addr)
}

func mvCNIConf(configDir, configFile, confName string) error {
Expand Down
29 changes: 28 additions & 1 deletion cmd/ovn_monitor/ovn_monitor.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
package ovn_monitor

import (
"fmt"
"net/http"
"os"
"strings"
"time"

"github.com/prometheus/client_golang/prometheus/promhttp"
"k8s.io/klog/v2"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
ovn "github.com/kubeovn/kube-ovn/pkg/ovnmonitor"
"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/versions"
Expand All @@ -29,5 +34,27 @@ func CmdMain() {

http.Handle(config.MetricsPath, promhttp.Handler())
klog.Infoln("Listening on", config.ListenAddress)
klog.Fatal(http.ListenAndServe(config.ListenAddress, nil))

// conform to Gosec G114
// https://github.com/securego/gosec#available-rules

addr := config.ListenAddress
if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
podIpsEnv := os.Getenv("POD_IPS")
podIps := strings.Split(podIpsEnv, ",")
// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
// so not support bind local ip when in dual mode
if len(podIps) == 1 {
addr = fmt.Sprintf("%s:10661", podIps[0])
if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
addr = fmt.Sprintf("[%s]:10661", podIps[0])
}
}
}

server := &http.Server{
Addr: addr,
ReadHeaderTimeout: 3 * time.Second,
}
util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", config.ListenAddress)
}
31 changes: 31 additions & 0 deletions dist/images/install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true}
IFACE=${IFACE:-}
# Specifies the name of the dpdk tunnel iface.
DPDK_TUNNEL_IFACE=${DPDK_TUNNEL_IFACE:-br-phy}
ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-true}

CNI_CONF_DIR="/etc/cni/net.d"
CNI_BIN_DIR="/opt/cni/bin"
Expand Down Expand Up @@ -1639,6 +1640,12 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: ENABLE_BIND_LOCAL_IP
value: "$ENABLE_BIND_LOCAL_IP"
resources:
requests:
cpu: 300m
Expand Down Expand Up @@ -2127,6 +2134,12 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: ENABLE_BIND_LOCAL_IP
value: "$ENABLE_BIND_LOCAL_IP"
resources:
requests:
cpu: 300m
Expand Down Expand Up @@ -2616,6 +2629,12 @@ spec:
fieldPath: spec.nodeName
- name: OVN_DB_IPS
value: $addresses
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: ENABLE_BIND_LOCAL_IP
value: "$ENABLE_BIND_LOCAL_IP"
volumeMounts:
- mountPath: /etc/localtime
name: localtime
Expand Down Expand Up @@ -2738,6 +2757,12 @@ spec:
value: $MODULES
- name: RPMS
value: $RPMS
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: ENABLE_BIND_LOCAL_IP
value: "$ENABLE_BIND_LOCAL_IP"
volumeMounts:
- name: host-modules
mountPath: /lib/modules
Expand Down Expand Up @@ -3013,6 +3038,12 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: ENABLE_BIND_LOCAL_IP
value: "$ENABLE_BIND_LOCAL_IP"
resources:
requests:
cpu: 200m
Expand Down
24 changes: 18 additions & 6 deletions dist/images/ovn-is-leader.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,24 @@ ovn-ctl status_northd
ovn-ctl status_ovnnb
ovn-ctl status_ovnsb

BIND_LOCAL_ADDR=127.0.0.1
if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
POD_IPS_LIST=(${POD_IPS//,/ })
if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then
if [[ $POD_IP =~ .*:.* ]]; then
BIND_LOCAL_ADDR=[${POD_IP}] #ipv6
else
BIND_LOCAL_ADDR=${POD_IP} #ipv4
fi
fi
fi

# For data consistency, only store leader address in endpoint
# Store ovn-nb leader to svc kube-system/ovn-nb
if [[ "$ENABLE_SSL" == "false" ]]; then
nb_leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
nb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
else
nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
fi

if [[ $nb_leader =~ "true" ]]
Expand All @@ -34,9 +46,9 @@ fi

# Store ovn-sb leader to svc kube-system/ovn-sb
if [[ "$ENABLE_SSL" == "false" ]]; then
sb_leader=$(ovsdb-client query tcp:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
sb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
else
sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
fi

if [[ $sb_leader =~ "true" ]]
Expand All @@ -51,9 +63,9 @@ then
if [ "$northd_leader" == "" ]; then
# no available northd leader try to release the lock
if [[ "$ENABLE_SSL" == "false" ]]; then
ovsdb-client -v -t 1 steal tcp:127.0.0.1:6642 ovn_northd
ovsdb-client -v -t 1 steal tcp:$BIND_LOCAL_ADDR:6642 ovn_northd
else
ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:127.0.0.1:6642 ovn_northd
ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:$BIND_LOCAL_ADDR:6642 ovn_northd
fi
fi
fi
Expand Down
28 changes: 18 additions & 10 deletions dist/images/start-db.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ DB_NB_PORT=${DB_NB_PORT:-6641}
DB_SB_ADDR=${DB_SB_ADDR:-::}
DB_SB_PORT=${DB_SB_PORT:-6642}
ENABLE_SSL=${ENABLE_SSL:-false}
ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-false}
BIND_LOCAL_ADDR=[::]
if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
POD_IPS_LIST=(${POD_IPS//,/ })
if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then
BIND_LOCAL_ADDR="[${POD_IP}]"
fi
fi

. /usr/share/openvswitch/scripts/ovs-lib || exit 1

Expand Down Expand Up @@ -177,8 +185,8 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
--db-sb-create-insecure-remote=yes \
--db-nb-cluster-local-addr="[${POD_IP}]" \
--db-sb-cluster-local-addr="[${POD_IP}]" \
--db-nb-addr=[::] \
--db-sb-addr=[::] \
--db-nb-addr=$BIND_LOCAL_ADDR \
--db-sb-addr=$BIND_LOCAL_ADDR \
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
start_northd
Expand Down Expand Up @@ -222,8 +230,8 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
--db-sb-cluster-local-addr="[${POD_IP}]" \
--db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
--db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
--db-nb-addr=[::] \
--db-sb-addr=[::] \
--db-nb-addr=$BIND_LOCAL_ADDR \
--db-sb-addr=$BIND_LOCAL_ADDR \
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
start_northd
Expand Down Expand Up @@ -277,16 +285,16 @@ else
--ovn-northd-ssl-ca-cert=/var/run/tls/cacert \
--db-nb-cluster-local-addr="[${POD_IP}]" \
--db-sb-cluster-local-addr="[${POD_IP}]" \
--db-nb-addr=[::] \
--db-sb-addr=[::] \
--db-nb-addr=$BIND_LOCAL_ADDR \
--db-sb-addr=$BIND_LOCAL_ADDR \
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
start_northd
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":[::]
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":$BIND_LOCAL_ADDR
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:use_logical_dp_groups=true

ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":[::]
ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":$BIND_LOCAL_ADDR
ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
else
# get leader if cluster exists
Expand Down Expand Up @@ -328,8 +336,8 @@ else
--db-sb-cluster-local-addr="[${POD_IP}]" \
--db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
--db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
--db-nb-addr=[::] \
--db-sb-addr=[::] \
--db-nb-addr=$BIND_LOCAL_ADDR \
--db-sb-addr=$BIND_LOCAL_ADDR \
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
start_northd
Expand Down

0 comments on commit 7dba66c

Please sign in to comment.