feat: ko support cluster operations status/kick/backup

(cherry picked from commit bdf9184)
kubeovn · Jul 7, 2021 · 498c7dd · 498c7dd
1 parent d812c74
commit 498c7dd
Show file tree

Hide file tree

Showing 4 changed files with 232 additions and 2 deletions.
diff --git a/dist/images/install-pre-1.16.sh b/dist/images/install-pre-1.16.sh
@@ -22,7 +22,7 @@ PINGER_EXTERNAL_DOMAIN="alauda.cn"         # Pinger check external domain probe
 if [ "$IPv6" = "true" ]; then
   POD_CIDR="fd00:10:16::/64"                # Do NOT overlap with NODE/SVC/JOIN CIDR
   POD_GATEWAY="fd00:10:16::1"
-  SVC_CIDR="fd00:10:96::/112"                # Do NOT overlap with NODE/POD/JOIN CIDR
+  SVC_CIDR="fd00:10:96::/112"               # Do NOT overlap with NODE/POD/JOIN CIDR
   JOIN_CIDR="fd00:100:64::/64"              # Do NOT overlap with NODE/POD/SVC CIDR
   PINGER_EXTERNAL_ADDRESS="2400:3200::1"
   PINGER_EXTERNAL_DOMAIN="google.com"
@@ -2062,6 +2062,7 @@ OVN_SB_POD=
 showHelp(){
   echo "kubectl ko {subcommand} [option...]"
   echo "Available Subcommands:"
+  echo "  [nb|sb] [status|kick|backup]     ovn-db operations show cluster status, kick stale server or backup database"
   echo "  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl"
   echo "  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl"
   echo "  vsctl {nodeName} [ovs-vsctl options ...]   invoke ovs-vsctl on selected node"
@@ -2206,11 +2207,28 @@ xxctl(){
   kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
 }
 
+checkLeader(){
+  component="$1"; shift
+  count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l)
+  if [ $count -eq 0 ]; then
+    echo "no ovn-$component exists !!"
+    exit 1
+  fi
+
+  if [ $count -gt 1 ]; then
+    echo "ovn-$component has more than one leader !!"
+    exit 1
+  fi
+
+  echo "ovn-$component leader check ok"
+}
+
 diagnose(){
   kubectl get crd vpcs.kubeovn.io
   kubectl get crd vpc-nat-gateways.kubeovn.io
   kubectl get crd subnets.kubeovn.io
   kubectl get crd ips.kubeovn.io
+  kubectl get crd vlans.kubeovn.io
   kubectl get svc kube-dns -n kube-system
   kubectl get svc kubernetes -n default
   kubectl get sa -n kube-system ovn
@@ -2230,6 +2248,11 @@ diagnose(){
   checkDaemonSet kube-ovn-cni
   checkDaemonSet ovs-ovn
   checkDeployment coredns
+
+  checkLeader nb
+  checkLeader sb
+  checkLeader northd
+
   type="$1"
   case $type in
     all)
@@ -2339,6 +2362,53 @@ checkKubeProxy(){
   else
     checkDaemonSet kube-proxy
   fi
+  echo "kube-proxy ready"
+}
+
+dbtool(){
+  suffix=$(date +%m%d%H%M%s)
+  component="$1"; shift
+  action="$1"; shift
+  case $component in
+    nb)
+      case $action in
+        status)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
+          ;;
+        kick)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1"
+          ;;
+        backup)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db
+          kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup
+          echo "backup $component to $(pwd)/ovnnb_db.$suffix.backup"
+          ;;
+        *)
+          echo "unknown action $action"
+      esac
+      ;;
+    sb)
+      case $action in
+        status)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
+          ;;
+        kick)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1"
+          ;;
+        backup)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db
+          kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup
+          echo "backup $component to $(pwd)/ovnsb_db.$suffix.backup"
+          ;;
+        *)
+          echo "unknown action $action"
+      esac
+      ;;
+    *)
+      echo "unknown subcommand $component"
+  esac
 }
 
 if [ $# -lt 1 ]; then
@@ -2360,6 +2430,9 @@ case $subcommand in
   vsctl|ofctl|dpctl|appctl)
     xxctl "$subcommand" "$@"
     ;;
+  nb|sb)
+    dbtool "$subcommand" "$@"
+    ;;
   tcpdump)
     tcpdump "$@"
     ;;

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -549,7 +549,6 @@ spec:
     kind: Subnet
     shortNames:
       - subnet
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -2113,6 +2112,7 @@ OVN_SB_POD=
 showHelp(){
   echo "kubectl ko {subcommand} [option...]"
   echo "Available Subcommands:"
+  echo "  [nb|sb] [status|kick|backup]     ovn-db operations show cluster status, kick stale server or backup database"
   echo "  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl"
   echo "  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl"
   echo "  vsctl {nodeName} [ovs-vsctl options ...]   invoke ovs-vsctl on selected node"
@@ -2257,11 +2257,28 @@ xxctl(){
   kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
 }
 
+checkLeader(){
+  component="$1"; shift
+  count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l)
+  if [ $count -eq 0 ]; then
+    echo "no ovn-$component exists !!"
+    exit 1
+  fi
+
+  if [ $count -gt 1 ]; then
+    echo "ovn-$component has more than one leader !!"
+    exit 1
+  fi
+
+  echo "ovn-$component leader check ok"
+}
+
 diagnose(){
   kubectl get crd vpcs.kubeovn.io
   kubectl get crd vpc-nat-gateways.kubeovn.io
   kubectl get crd subnets.kubeovn.io
   kubectl get crd ips.kubeovn.io
+  kubectl get crd vlans.kubeovn.io
   kubectl get svc kube-dns -n kube-system
   kubectl get svc kubernetes -n default
   kubectl get sa -n kube-system ovn
@@ -2281,6 +2298,11 @@ diagnose(){
   checkDaemonSet kube-ovn-cni
   checkDaemonSet ovs-ovn
   checkDeployment coredns
+
+  checkLeader nb
+  checkLeader sb
+  checkLeader northd
+
   type="$1"
   case $type in
     all)
@@ -2390,6 +2412,53 @@ checkKubeProxy(){
   else
     checkDaemonSet kube-proxy
   fi
+  echo "kube-proxy ready"
+}
+
+dbtool(){
+  suffix=$(date +%m%d%H%M%s)
+  component="$1"; shift
+  action="$1"; shift
+  case $component in
+    nb)
+      case $action in
+        status)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
+          ;;
+        kick)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1"
+          ;;
+        backup)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db
+          kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup
+          echo "backup $component to $(pwd)/ovnnb_db.$suffix.backup"
+          ;;
+        *)
+          echo "unknown action $action"
+      esac
+      ;;
+    sb)
+      case $action in
+        status)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
+          ;;
+        kick)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1"
+          ;;
+        backup)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db
+          kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup
+          echo "backup $component to $(pwd)/ovnsb_db.$suffix.backup"
+          ;;
+        *)
+          echo "unknown action $action"
+      esac
+      ;;
+    *)
+      echo "unknown subcommand $component"
+  esac
 }
 
 if [ $# -lt 1 ]; then
@@ -2411,6 +2480,9 @@ case $subcommand in
   vsctl|ofctl|dpctl|appctl)
     xxctl "$subcommand" "$@"
     ;;
+  nb|sb)
+    dbtool "$subcommand" "$@"
+    ;;
   tcpdump)
     tcpdump "$@"
     ;;

diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko
@@ -8,6 +8,7 @@ OVN_SB_POD=
 showHelp(){
   echo "kubectl ko {subcommand} [option...]"
   echo "Available Subcommands:"
+  echo "  [nb|sb] [status|kick|backup]     ovn-db operations show cluster status, kick stale server or backup database"
   echo "  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl"
   echo "  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl"
   echo "  vsctl {nodeName} [ovs-vsctl options ...]   invoke ovs-vsctl on selected node"
@@ -152,6 +153,22 @@ xxctl(){
   kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
 }
 
+checkLeader(){
+  component="$1"; shift
+  count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l)
+  if [ $count -eq 0 ]; then
+    echo "no ovn-$component exists !!"
+    exit 1
+  fi
+
+  if [ $count -gt 1 ]; then
+    echo "ovn-$component has more than one leader !!"
+    exit 1
+  fi
+
+  echo "ovn-$component leader check ok"
+}
+
 diagnose(){
   kubectl get crd vpcs.kubeovn.io
   kubectl get crd vpc-nat-gateways.kubeovn.io
@@ -176,6 +193,11 @@ diagnose(){
   checkDaemonSet kube-ovn-cni
   checkDaemonSet ovs-ovn
   checkDeployment coredns
+
+  checkLeader nb
+  checkLeader sb
+  checkLeader northd
+
   type="$1"
   case $type in
     all)
@@ -287,6 +309,52 @@ checkKubeProxy(){
   echo "kube-proxy ready"
 }
 
+dbtool(){
+  suffix=$(date +%m%d%H%M%s)
+  component="$1"; shift
+  action="$1"; shift
+  case $component in
+    nb)
+      case $action in
+        status)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
+          ;;
+        kick)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1"
+          ;;
+        backup)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /tmp/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db
+          kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/tmp/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /tmp/ovnnb_db.$suffix.backup
+          echo "backup ovn-$component db to $(pwd)/ovnnb_db.$suffix.backup"
+          ;;
+        *)
+          echo "unknown action $action"
+      esac
+      ;;
+    sb)
+      case $action in
+        status)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
+          ;;
+        kick)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1"
+          ;;
+        backup)
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /tmp/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db
+          kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/tmp/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /tmp/ovnsb_db.$suffix.backup
+          echo "backup ovn-$component db to $(pwd)/ovnsb_db.$suffix.backup"
+          ;;
+        *)
+          echo "unknown action $action"
+      esac
+      ;;
+    *)
+      echo "unknown subcommand $component"
+  esac
+}
+
 if [ $# -lt 1 ]; then
   showHelp
   exit 0
@@ -306,6 +374,9 @@ case $subcommand in
   vsctl|ofctl|dpctl|appctl)
     xxctl "$subcommand" "$@"
     ;;
+  nb|sb)
+    dbtool "$subcommand" "$@"
+    ;;
   tcpdump)
     tcpdump "$@"
     ;;

diff --git a/test/e2e/kubectl-ko/ko.go b/test/e2e/kubectl-ko/ko.go
@@ -86,4 +86,18 @@ var _ = Describe("[kubectl-ko]", func() {
 		output, err = exec.Command("kubectl", "ko", "trace", fmt.Sprintf("kube-system/%s", pod.Name), "114.114.114.114", "udp", "53").CombinedOutput()
 		Expect(err).NotTo(HaveOccurred(), string(output))
 	})
+
+	It("nb/sb operation", func() {
+		output, err := exec.Command("kubectl", "ko", "nb", "status").CombinedOutput()
+		Expect(err).NotTo(HaveOccurred(), string(output))
+
+		output, err = exec.Command("kubectl", "ko", "sb", "status").CombinedOutput()
+		Expect(err).NotTo(HaveOccurred(), string(output))
+
+		output, err = exec.Command("kubectl", "ko", "nb", "backup").CombinedOutput()
+		Expect(err).NotTo(HaveOccurred(), string(output))
+
+		output, err = exec.Command("kubectl", "ko", "sb", "backup").CombinedOutput()
+		Expect(err).NotTo(HaveOccurred(), string(output))
+	})
 })