update script to add restore plugin cmd

kubeovn · Mar 11, 2022 · 7397db2 · 7397db2
1 parent 46eb49a
commit 7397db2
Show file tree

Hide file tree

Showing 5 changed files with 314 additions and 47 deletions.
diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -2890,7 +2890,7 @@ dbtool(){
           done
 
           echo "backup nb db file"
-          docker run -it -v /etc/origin/ovn:/etc/ovn $REGISTRY/kube-ovn:$VERSION bash -c "ovsdb-tool cluster-to-standalone  /etc/ovn/ovnnb_db_standalone.db  /etc/ovn/ovnnb_db.db"
+          docker run -it -v /etc/origin/ovn:/etc/ovn $REGISTRY/kube-ovn:$KUBE_OVN_VERSION bash -c "ovsdb-tool cluster-to-standalone  /etc/ovn/ovnnb_db_standalone.db  /etc/ovn/ovnnb_db.db"
 
           # mv all db files
           for pod in ${podNameArray[@]}

diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko
@@ -462,7 +462,7 @@ dbtool(){
           done
 
           echo "backup nb db file"
-          docker run -it -v /etc/origin/ovn:/etc/ovn $REGISTRY/kube-ovn:$VERSION bash -c "ovsdb-tool cluster-to-standalone  /etc/ovn/ovnnb_db_standalone.db  /etc/ovn/ovnnb_db.db"
+          docker run -it -v /etc/origin/ovn:/etc/ovn $REGISTRY/kube-ovn:$KUBE_OVN_VERSION bash -c "ovsdb-tool cluster-to-standalone  /etc/ovn/ovnnb_db_standalone.db  /etc/ovn/ovnnb_db.db"
 
           # mv all db files
           for pod in ${podNameArray[@]}

diff --git a/dist/images/update/1.7-1.8.2.sh → dist/images/update/1.7-1.8.3.sh b/dist/images/update/1.7-1.8.2.sh → dist/images/update/1.7-1.8.3.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eo pipefail
 
-IMAGE=kubeovn/kube-ovn:v1.8.2
+IMAGE=kubeovn/kube-ovn:v1.8.3
 
 echo "[Step 0/8] Update CRD"
 cat <<EOF > kube-ovn-crd-1.8.yaml
@@ -828,6 +828,7 @@ echo ""
 echo "[Step 7/8] Update kube-ovn-monitor"
 kubectl set image deployment/kube-ovn-monitor -n kube-system kube-ovn-monitor="$IMAGE"
 kubectl patch deployment/kube-ovn-monitor -n kube-system --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/hostNetwork", "value": true}]'
+kubectl patch deployment/kube-ovn-monitor -n kube-system --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/livenessProbe/exec/command", "value": [cat, /var/run/ovn/ovnnb_db.pid]}]'
 kubectl rollout status deployment/kube-ovn-monitor -n kube-system
 echo "-------------------------------"
 echo ""
@@ -841,11 +842,13 @@ set -euo pipefail
 KUBE_OVN_NS=kube-system
 OVN_NB_POD=
 OVN_SB_POD=
+KUBE_OVN_VERSION=
+REGISTRY="kubeovn"
 
 showHelp(){
   echo "kubectl ko {subcommand} [option...]"
   echo "Available Subcommands:"
-  echo "  [nb|sb] [status|kick|backup]     ovn-db operations show cluster status, kick stale server or backup database"
+  echo "  [nb|sb] [status|kick|backup|dbstatus|restore]     ovn-db operations show cluster status, kick stale server, backup database, get db consistency status or restore ovn nb db when met 'inconsistent data' error"
   echo "  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl"
   echo "  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl"
   echo "  vsctl {nodeName} [ovs-vsctl options ...]   invoke ovs-vsctl on the specified node"
@@ -855,6 +858,7 @@ showHelp(){
   echo "  tcpdump {namespace/podname} [tcpdump options ...]     capture pod traffic"
   echo "  trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]    trace ovn microflow of specific packet"
   echo "  diagnose {all|node} [nodename]    diagnose connectivity of all nodes or a specific node"
+  echo "  reload restart all kube-ovn components"
 }
 
 tcpdump(){
@@ -922,22 +926,30 @@ trace(){
     proto="6"
   fi
 
-  podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}"))
-  mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
-  ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
   hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
-  nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
-
   if [ "$hostNetwork" = "true" ]; then
     echo "Can not trace host network pod"
     exit 1
   fi
 
+  ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
   if [ -z "$ls" ]; then
     echo "pod address not ready"
     exit 1
   fi
 
+  podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}"))
+  if [ ${#podIPs[@]} -eq 0 ]; then
+    podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g'))
+    if [ ${#podIPs[@]} -eq 0 ]; then
+      echo "pod address not ready"
+      exit 1
+    fi
+  fi
+
+  mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
+  nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
+
   podIP=""
   for ip in ${podIPs[@]}; do
     if [ "$af" = "4" ]; then
@@ -957,7 +969,9 @@ trace(){
   fi
 
   gwMac=""
-  if [ ! -z "$(kubectl get subnet $ls -o jsonpath={.spec.vlan})" ]; then
+  vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
+  logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
+  if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
     gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
     if [[ "$gateway" =~ .*,.* ]]; then
       if [ "$af" = "4" ]; then
@@ -1093,16 +1107,22 @@ diagnose(){
   kubectl get crd ips.kubeovn.io
   kubectl get crd vlans.kubeovn.io
   kubectl get crd provider-networks.kubeovn.io
-  kubectl get svc kube-dns -n kube-system
+  set +eu
+  if ! kubectl get svc kube-dns -n kube-system ; then
+     echo "Warning: kube-dns doesn't exist, maybe there is coredns service."
+  fi
+  set -eu
   kubectl get svc kubernetes -n default
   kubectl get sa -n kube-system ovn
   kubectl get clusterrole system:ovn
   kubectl get clusterrolebinding ovn
 
   kubectl get no -o wide
   kubectl ko nbctl show
+  kubectl ko nbctl lr-policy-list ovn-cluster
   kubectl ko nbctl lr-route-list ovn-cluster
   kubectl ko nbctl ls-lb-list ovn-default
+  kubectl ko nbctl list address_set
   kubectl ko nbctl list acl
   kubectl ko sbctl show
 
@@ -1184,6 +1204,12 @@ getOvnCentralPod(){
       exit 1
     fi
     OVN_SB_POD=$SB_POD
+    VERSION=$(kubectl  -n kube-system get pods -l ovn-sb-leader=true -o yaml | grep  "image: $REGISTRY/kube-ovn:" | head -n 1 | awk -F ':' '{print $3}')
+    if [ -z "$VERSION" ]; then
+          echo "kubeovn version not exists"
+          exit 1
+        fi
+    KUBE_OVN_VERSION=$VERSION
 }
 
 checkDaemonSet(){
@@ -1241,16 +1267,66 @@ dbtool(){
       case $action in
         status)
           kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl ovsdb-server/get-db-storage-status OVN_Northbound
           ;;
         kick)
           kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1"
           ;;
         backup)
-          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /tmp/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db
-          kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/tmp/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup
-          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /tmp/ovnnb_db.$suffix.backup
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db
+          kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup
           echo "backup ovn-$component db to $(pwd)/ovnnb_db.$suffix.backup"
           ;;
+        dbstatus)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-appctl -t /var/run/ovn/ovnnb_db.ctl ovsdb-server/get-db-storage-status OVN_Northbound
+          ;;
+        restore)
+          # set ovn-central replicas to 0
+          replicas=$(kubectl get deployment -n $KUBE_OVN_NS ovn-central -o jsonpath={.spec.replicas})
+          kubectl scale deployment -n $KUBE_OVN_NS ovn-central --replicas=0
+          echo "ovn-central original replicas is $replicas"
+
+          # backup ovn-nb db
+          declare nodeIpArray
+          declare podNameArray
+          nodeIps=`kubectl get node -lkube-ovn/role=master -o wide | grep -v "INTERNAL-IP" | awk '{print $6}'`
+          firstIP=${nodeIps[0]}
+          podNames=`kubectl get pod -n $KUBE_OVN_NS | grep ovs-ovn | awk '{print $1}'`
+          echo "first nodeIP is $firstIP"
+
+          i=0
+          for nodeIp in $nodeIps
+          do
+            for pod in $podNames
+            do
+              hostip=$(kubectl get pod -n $KUBE_OVN_NS $pod -o jsonpath={.status.hostIP})
+              if [ $nodeIp = $hostip ]; then
+                nodeIpArray[$i]=$nodeIp
+                podNameArray[$i]=$pod
+                i=`expr $i + 1`
+                echo "ovs-ovn pod on node $nodeIp is $pod"
+                break
+              fi
+            done
+          done
+
+          echo "backup nb db file"
+          docker run -it -v /etc/origin/ovn:/etc/ovn $REGISTRY/kube-ovn:$KUBE_OVN_VERSION bash -c "ovsdb-tool cluster-to-standalone  /etc/ovn/ovnnb_db_standalone.db  /etc/ovn/ovnnb_db.db"
+
+          # mv all db files
+          for pod in ${podNameArray[@]}
+          do
+            kubectl exec -it -n $KUBE_OVN_NS $pod -- mv /etc/ovn/ovnnb_db.db /tmp
+            kubectl exec -it -n $KUBE_OVN_NS $pod -- mv /etc/ovn/ovnsb_db.db /tmp
+          done
+
+          # restore db and replicas
+          echo "restore nb db file, operate in pod ${podNameArray[0]}"
+          kubectl exec -it -n $KUBE_OVN_NS ${podNameArray[0]} -- mv /etc/ovn/ovnnb_db_standalone.db /etc/ovn/ovnnb_db.db
+          kubectl scale deployment -n $KUBE_OVN_NS ovn-central --replicas=$replicas
+          echo "finish restore nb db file and ovn-central replicas"
+          ;;
         *)
           echo "unknown action $action"
       esac
@@ -1259,16 +1335,23 @@ dbtool(){
       case $action in
         status)
           kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl ovsdb-server/get-db-storage-status OVN_Southbound
           ;;
         kick)
           kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1"
           ;;
         backup)
-          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /tmp/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db
-          kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/tmp/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup
-          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /tmp/ovnsb_db.$suffix.backup
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db
+          kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup
+          kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup
           echo "backup ovn-$component db to $(pwd)/ovnsb_db.$suffix.backup"
           ;;
+        dbstatus)
+          kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-appctl -t /var/run/ovn/ovnsb_db.ctl ovsdb-server/get-db-storage-status OVN_Southbound
+          ;;
+        restore)
+          echo "restore cmd is only used for nb db"
+          ;;
         *)
           echo "unknown action $action"
       esac
@@ -1278,6 +1361,20 @@ dbtool(){
   esac
 }
 
+reload(){
+  kubectl delete pod -n kube-system -l app=ovn-central
+  kubectl rollout status deployment/ovn-central -n kube-system
+  kubectl delete pod -n kube-system -l app=ovs
+  kubectl delete pod -n kube-system -l app=kube-ovn-controller
+  kubectl rollout status deployment/kube-ovn-controller -n kube-system
+  kubectl delete pod -n kube-system -l app=kube-ovn-cni
+  kubectl rollout status daemonset/kube-ovn-cni -n kube-system
+  kubectl delete pod -n kube-system -l app=kube-ovn-pinger
+  kubectl rollout status daemonset/kube-ovn-pinger -n kube-system
+  kubectl delete pod -n kube-system -l app=kube-ovn-monitor
+  kubectl rollout status deployment/kube-ovn-monitor -n kube-system
+}
+
 if [ $# -lt 1 ]; then
   showHelp
   exit 0
@@ -1309,8 +1406,11 @@ case $subcommand in
   diagnose)
     diagnose "$@"
     ;;
+  reload)
+    reload
+    ;;
   *)
-    showHelp
+  showHelp
     ;;
 esac
 EOF