fix: release norhtd lock when power off

When the node that runs northd power off, the lock to southbound will not released. Use ep to store the norhtd leader and try to release lock if no alive northd.
kubeovn · Apr 14, 2021 · 9e82ca3 · 9e82ca3
1 parent b72e9d5
commit 9e82ca3
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 0 deletions.
diff --git a/dist/images/install-pre-1.16.sh b/dist/images/install-pre-1.16.sh
@@ -689,6 +689,24 @@ spec:
     ovn-sb-leader: "true"
   sessionAffinity: None
 
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: ovn-northd
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-northd
+      protocol: TCP
+      port: 6643
+      targetPort: 6643
+  type: ClusterIP
+  selector:
+    app: ovn-central
+    ovn-northd-leader: "true"
+  sessionAffinity: None
+
 ---
 kind: Service
 apiVersion: v1
@@ -1189,6 +1207,23 @@ spec:
 ---
 kind: Service
 apiVersion: v1
+metadata:
+  name: ovn-northd
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-northd
+      protocol: TCP
+      port: 6643
+      targetPort: 6643
+  type: ClusterIP
+  selector:
+    app: ovn-central
+    ovn-northd-leader: "true"
+  sessionAffinity: None
+---
+kind: Service
+apiVersion: v1
 metadata:
   name: kube-ovn-monitor
   namespace: kube-system

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -729,6 +729,24 @@ spec:
     ovn-sb-leader: "true"
   sessionAffinity: None
 
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: ovn-northd
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-northd
+      protocol: TCP
+      port: 6643
+      targetPort: 6643
+  type: ClusterIP
+  selector:
+    app: ovn-central
+    ovn-northd-leader: "true"
+  sessionAffinity: None
+
 ---
 kind: Service
 apiVersion: v1
@@ -1239,6 +1257,23 @@ spec:
 ---
 kind: Service
 apiVersion: v1
+metadata:
+  name: ovn-northd
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-northd
+      protocol: TCP
+      port: 6643
+      targetPort: 6643
+  type: ClusterIP
+  selector:
+    app: ovn-central
+    ovn-northd-leader: "true"
+  sessionAffinity: None
+---
+kind: Service
+apiVersion: v1
 metadata:
   name: kube-ovn-monitor
   namespace: kube-system

diff --git a/dist/images/ovn-is-leader.sh b/dist/images/ovn-is-leader.sh
@@ -9,26 +9,50 @@ ovn-ctl status_ovnnb
 ovn-ctl status_ovnsb
 
 # For data consistency, only store leader address in endpoint
+# Store ovn-nb leader to svc kube-system/ovn-nb
 if [[ "$ENABLE_SSL" == "false" ]]; then
   nb_leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 else
   nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 fi
+
 if [[ $nb_leader =~ "true" ]]
 then
    kubectl label --overwrite pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-nb-leader=true
 else
   kubectl label pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-nb-leader-
 fi
 
+# Store ovn-northd leader to svc kube-system/ovn-northd
+northd_status=$(ovs-appctl -t /var/run/ovn/ovn-northd.$(cat /var/run/ovn/ovn-northd.pid).ctl status)
+if [[ $northd_status =~ "active" ]]
+then
+   kubectl label --overwrite pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-northd-leader=true
+else
+  kubectl label pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-northd-leader-
+fi
+
+# Store ovn-sb leader to svc kube-system/ovn-sb
 if [[ "$ENABLE_SSL" == "false" ]]; then
   sb_leader=$(ovsdb-client query tcp:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 else
   sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 fi
+
 if [[ $sb_leader =~ "true" ]]
 then
    kubectl label --overwrite pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-sb-leader=true
+   northd_leader=$(kubectl get ep -n kube-system ovn-northd -o jsonpath={.subsets\[0\].addresses\[0\].ip})
+   if [ "$northd_leader" == "" ]; then
+      # no available northd leader try to release the lock
+      set +e
+      if [[ "$ENABLE_SSL" == "false" ]]; then
+        ovsdb-client -v -t 1 steal tcp:127.0.0.1:6642  ovn_northd
+      else
+        ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:127.0.0.1:6642  ovn_northd
+      fi
+      set -e
+    fi
 else
   kubectl label pod "$POD_NAME" -n "$POD_NAMESPACE" ovn-sb-leader-
 fi
diff --git a/yamls/ovn-ha.yaml b/yamls/ovn-ha.yaml
@@ -148,6 +148,23 @@ spec:
     ovn-sb-leader: "true"
   sessionAffinity: None
 ---
+kind: Service
+apiVersion: v1
+metadata:
+  name: ovn-northd
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-northd
+      protocol: TCP
+      port: 6643
+      targetPort: 6643
+  type: ClusterIP
+  selector:
+    app: ovn-central
+    ovn-northd-leader: "true"
+  sessionAffinity: None
+---
 kind: Deployment
 apiVersion: apps/v1
 metadata:

diff --git a/yamls/ovn.yaml b/yamls/ovn.yaml
@@ -168,6 +168,23 @@ spec:
 ---
 kind: Service
 apiVersion: v1
+metadata:
+  name: ovn-northd
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-northd
+      protocol: TCP
+      port: 6643
+      targetPort: 6643
+  type: ClusterIP
+  selector:
+    app: ovn-central
+    ovn-northd-leader: "true"
+  sessionAffinity: None
+---
+kind: Service
+apiVersion: v1
 metadata:
   name: kube-ovn-monitor
   namespace:  kube-system