fix: ovn-cluster mode

kubeovn · Dec 5, 2019 · a5eb5e7 · a5eb5e7
1 parent a6f0dd1
commit a5eb5e7
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 38 deletions.
diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko
@@ -2,7 +2,8 @@
 set -euo pipefail
 
 KUBE_OVN_NS=kube-ovn
-CENTRAL_POD=
+OVN_NB_POD=
+OVN_SB_POD=
 
 showHelp(){
   echo "kubectl ko {subcommand} [option...]"
@@ -83,7 +84,7 @@ trace(){
     exit 1
   fi
 
-  gwMac=$(kubectl exec -it $CENTRAL_POD -n $KUBE_OVN_NS -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name=ovn-cluster-"$ls" | tr -d '\r')
+  gwMac=$(kubectl exec -it $OVN_NB_POD -n $KUBE_OVN_NS -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name=ovn-cluster-"$ls" | tr -d '\r')
 
   if [ -z "$gwMac" ]; then
     echo "get gw mac failed"
@@ -101,11 +102,11 @@ trace(){
   case $type in
     icmp)
       set -x
-      kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst"
+      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst"
       ;;
     tcp|udp)
       set -x
-      kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4"
+      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4"
       ;;
     *)
       echo "type $type not supported"
@@ -156,12 +157,18 @@ diagnose(){
 }
 
 getOvnCentralPod(){
-    centralPod=$(kubectl get pod -n $KUBE_OVN_NS | grep ovn-central | head -n 1 | awk '{print $1}')
-    if [ -z "$centralPod" ]; then
-      echo "ovn-central not exists"
+    NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-nb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}')
+    if [ -z "$NB_POD" ]; then
+      echo "nb leader not exists"
       exit 1
     fi
-    CENTRAL_POD=$centralPod
+    OVN_NB_POD=$NB_POD
+    SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-sb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}')
+    if [ -z "$SB_POD" ]; then
+      echo "nb leader not exists"
+      exit 1
+    fi
+    OVN_SB_POD=$SB_POD
 }
 
 if [ $# -lt 1 ]; then
@@ -175,10 +182,10 @@ getOvnCentralPod
 
 case $subcommand in
   nbctl)
-    kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-nbctl "$@"
+    kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -- ovn-nbctl "$@"
     ;;
   sbctl)
-    kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-sbctl "$@"
+    kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-sbctl "$@"
     ;;
   vsctl)
     vsctl "$@"

diff --git a/dist/images/ovn-is-leader.sh b/dist/images/ovn-is-leader.sh
@@ -8,10 +8,18 @@ ovn-ctl status_ovnnb
 ovn-ctl status_ovnsb
 
 # For data consistency, only store leader address in endpoint
-leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
-if [[ $leader =~ "true" ]]
+nb_leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+if [[ $nb_leader =~ "true" ]]
 then
-   kubectl label --overwrite pod $POD_NAME -n $POD_NAMESPACE ovn-db-leader=true
+   kubectl label --overwrite pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-nb-leader=true
 else
-  kubectl label pod $POD_NAME -n $POD_NAMESPACE ovn-db-leader-
+  kubectl label pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-nb-leader-
+fi
+
+sb_leader=$(ovsdb-client query tcp:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+if [[ $sb_leader =~ "true" ]]
+then
+   kubectl label --overwrite pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-sb-leader=true
+else
+  kubectl label pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-sb-leader-
 fi
diff --git a/dist/images/start-db.sh b/dist/images/start-db.sh
@@ -7,14 +7,29 @@ DB_SB_ADDR=${DB_SB_ADDR:-::}
 DB_SB_PORT=${DB_SB_PORT:-6642}
 
 function gen_conn_str {
-  t=$(echo -n ${NODE_IPS} | sed 's/[[:space:]]//g' | sed 's/,/ /g')
+  t=$(echo -n "${NODE_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g')
   x=$(for i in ${t}; do echo -n "tcp:$i:$1",; done| sed 's/,$//')
   echo "$x"
 }
 
-function get_first_node_ip {
-    t=$(echo -n ${NODE_IPS} | sed 's/[[:space:]]//g' | sed 's/,/ /g')
-    echo -n ${t} | cut -f 1 -d " "
+function get_leader_ip {
+  while true; do
+    leader=$(kubectl get ep -n "${POD_NAMESPACE}" | grep ovn-"$1" | awk '{print $2}' | cut -d ":" -f1)
+    if [ "$leader" == "<none>" ]; then
+      break
+    else
+      if [[ "$leader" != "${POD_IP}" ]]; then
+        echo "$leader"
+        return
+      else
+        echo "leader cannot be self, waiting new leader"
+        sleep 5
+      fi
+    fi
+  done
+    # If no available leader the first ip will be the leader
+    t=$(echo -n "${NODE_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g')
+    echo -n "${t}" | cut -f 1 -d " "
 }
 
 function quit {
@@ -28,40 +43,47 @@ if [[ -z "$NODE_IPS" ]]; then
 else
     /usr/share/openvswitch/scripts/ovn-ctl stop_northd
 
-    first_node_ip=$(get_first_node_ip)
-    if [[ "$first_node_ip" == "${POD_IP}" ]]; then
+    nb_leader_ip=$(get_leader_ip nb)
+    sb_leader_ip=$(get_leader_ip sb)
+    if [[ "$nb_leader_ip" == "${POD_IP}" ]]; then
         # Start ovn-northd, ovn-nb and ovn-sb
         /usr/share/openvswitch/scripts/ovn-ctl \
             --db-nb-create-insecure-remote=yes \
             --db-sb-create-insecure-remote=yes \
-            --db-nb-cluster-local-addr=${POD_IP} \
-            --db-sb-cluster-local-addr=${POD_IP} \
+            --db-nb-cluster-local-addr="${POD_IP}" \
+            --db-sb-cluster-local-addr="${POD_IP}" \
             --ovn-northd-nb-db=$(gen_conn_str 6641) \
             --ovn-northd-sb-db=$(gen_conn_str 6642) \
             start_northd
+
+        # ovn-nb and ovn-sb listen on tcp ports for ovn-controller to connect
+        ovn-nbctl set-connection ptcp:"${DB_NB_PORT}":["${DB_NB_ADDR}"]
+        ovn-sbctl set-connection ptcp:"${DB_SB_PORT}":["${DB_SB_ADDR}"]
+        ovn-sbctl set Connection . inactivity_probe=0
     else
-        while ! nc -z ${first_node_ip} ${DB_NB_PORT} </dev/null;
+        while ! nc -z "${nb_leader_ip}" "${DB_NB_PORT}" >/dev/null;
+        do
+            echo "sleep 5 seconds, waiting for ovn-nb ${nb_leader_ip}:${DB_NB_PORT} ready "
+            sleep 5;
+        done
+        while ! nc -z "${sb_leader_ip}" "${DB_SB_PORT}" >/dev/null;
         do
-            echo "sleep 5 seconds, waiting for ovn-nb ${first_node_ip}:${DB_NB_PORT} ready "
+            echo "sleep 5 seconds, waiting for ovn-sb ${sb_leader_ip}:${DB_NB_PORT} ready "
             sleep 5;
         done
+
         # Start ovn-northd, ovn-nb and ovn-sb
         /usr/share/openvswitch/scripts/ovn-ctl \
             --db-nb-create-insecure-remote=yes \
             --db-sb-create-insecure-remote=yes \
-            --db-nb-cluster-local-addr=${POD_IP} \
-            --db-sb-cluster-local-addr=${POD_IP} \
-            --db-nb-cluster-remote-addr=${first_node_ip} \
-            --db-sb-cluster-remote-addr=${first_node_ip} \
+            --db-nb-cluster-local-addr="${POD_IP}" \
+            --db-sb-cluster-local-addr="${POD_IP}" \
+            --db-nb-cluster-remote-addr="${nb_leader_ip}" \
+            --db-sb-cluster-remote-addr="${sb_leader_ip}" \
             --ovn-northd-nb-db=$(gen_conn_str 6641) \
             --ovn-northd-sb-db=$(gen_conn_str 6642) \
             start_northd
     fi
 fi
 
-# ovn-nb and ovn-sb listen on tcp ports for ovn-controller to connect
-ovn-nbctl set-connection ptcp:${DB_NB_PORT}:[${DB_NB_ADDR}]
-ovn-sbctl set-connection ptcp:${DB_SB_PORT}:[${DB_SB_ADDR}]
-ovn-sbctl set Connection . inactivity_probe=0
-
 tail -f /var/log/openvswitch/ovn-northd.log
diff --git a/yamls/ovn.yaml b/yamls/ovn.yaml
@@ -99,7 +99,7 @@ spec:
   type: ClusterIP
   selector:
     app: ovn-central
-    ovn-db-leader: "true"
+    ovn-nb-leader: "true"
   sessionAffinity: None
 
 ---
@@ -117,7 +117,7 @@ spec:
   type: ClusterIP
   selector:
     app: ovn-central
-    ovn-db-leader: "true"
+    ovn-sb-leader: "true"
   sessionAffinity: None
 
 ---
@@ -162,6 +162,9 @@ spec:
         - name: ovn-central
           image: "index.alauda.cn/alaudak8s/kube-ovn-db:v1.0.0-pre"
           imagePullPolicy: Always
+          securityContext:
+            capabilities:
+              add: ["SYS_NICE"]
           env:
             - name: POD_IP
               valueFrom:
@@ -179,9 +182,6 @@ spec:
             requests:
               cpu: 500m
               memory: 300Mi
-            limits:
-              cpu: 1
-              memory: 800Mi
           volumeMounts:
             - mountPath: /run/openvswitch
               name: host-run-ovs