fix cleanup.sh and uninstall.sh

kubeovn · Oct 25, 2021 · 9bb0cfc · 9bb0cfc
1 parent da422ff
commit 9bb0cfc
Show file tree

Hide file tree

Showing 4 changed files with 239 additions and 45 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -113,6 +113,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   single-iptables-e2e:
     needs: build
     name: 1-master-iptables-e2e
@@ -157,6 +162,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   single-vlan-e2e:
     needs: build
     name: 1-master-vlan-e2e
@@ -201,6 +211,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e-vlan
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   single-underlay-e2e-single-nic:
     needs: build
     name: 1-master-underlay-e2e-single-nic
@@ -245,6 +260,27 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e-underlay-single-nic
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+      
+      - name: Check Node Network
+        run: |
+          sh -c '
+            set -e
+            kubectl get no --no-headers | awk "{print \$1}" | while read node; do
+              docker inspect $node -f "{{.NetworkSettings.Networks.kind.IPAddress}}"
+            done | while read ip; do
+              docker run --rm --network kind kubeovn/kube-ovn:$(cat VERSION) ping -c1 -w1 $ip
+            done
+            kubectl get no --no-headers | awk "{print \$1}" | while read node; do
+              docker inspect $node -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"
+            done | while read ip; do
+              docker run --rm --network kind kubeovn/kube-ovn:$(cat VERSION) ping6 -c1 -w1 $ip
+            done
+          '
+
   single-node-e2e:
     needs: build
     name: 1-node-e2e
@@ -289,6 +325,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   ha-e2e:
     needs: build
     name: 3-master-e2e
@@ -368,6 +409,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e-ipv6
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   ipv6-vlan-e2e:
     needs: build
     name: ipv6-vlan-e2e
@@ -416,6 +462,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e-vlan-ipv6
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   ipv6-underlay-e2e-single-nic:
     needs: build
     name: ipv6-underlay-e2e-single-nic
@@ -464,6 +515,27 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e-underlay-single-nic
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
+      - name: Check Node Network
+        run: |
+          sh -c '
+            set -e
+            kubectl get no --no-headers | awk "{print \$1}" | while read node; do
+              docker inspect $node -f "{{.NetworkSettings.Networks.kind.IPAddress}}"
+            done | while read ip; do
+              docker run --rm --network kind kubeovn/kube-ovn:$(cat VERSION) ping -c1 -w1 $ip
+            done
+            kubectl get no --no-headers | awk "{print \$1}" | while read node; do
+              docker inspect $node -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"
+            done | while read ip; do
+              docker run --rm --network kind kubeovn/kube-ovn:$(cat VERSION) ping6 -c1 -w1 $ip
+            done
+          '
+
   dual-stack-e2e:
     needs: build
     name: dual-stack-e2e
@@ -512,6 +584,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   dual-stack-underlay-e2e-single-nic:
     needs: build
     name: dual-stack-underlay-e2e-single-nic
@@ -560,6 +637,27 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e-underlay-single-nic
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
+      - name: Check Node Network
+        run: |
+          sh -c '
+            set -e
+            kubectl get no --no-headers | awk "{print \$1}" | while read node; do
+              docker inspect $node -f "{{.NetworkSettings.Networks.kind.IPAddress}}"
+            done | while read ip; do
+              docker run --rm --network kind kubeovn/kube-ovn:$(cat VERSION) ping -c1 -w1 $ip
+            done
+            kubectl get no --no-headers | awk "{print \$1}" | while read node; do
+              docker inspect $node -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"
+            done | while read ip; do
+              docker run --rm --network kind kubeovn/kube-ovn:$(cat VERSION) ping6 -c1 -w1 $ip
+            done
+          '
+
   no-lb-e2e:
     needs: build
     name: disable-loadbalancer-e2e
@@ -606,6 +704,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   no-lb-iptables-e2e:
     needs: build
     name: disable-loadbalancer-iptables-e2e
@@ -652,6 +755,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   no-np-e2e:
     needs: build
     name: disable-network-policy-e2e
@@ -698,6 +806,11 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
+      - name: Cleanup
+        run: |
+          sh -c 'while :; do if [ $(kubectl get --no-headers subnet | wc -l) -eq 2 ]; then break; fi; sleep 5; done'
+          sh dist/images/cleanup.sh
+
   push:
     needs:
       - single-e2e

diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh
@@ -3,6 +3,7 @@ set -eu
 
 for subnet in $(kubectl get subnet -o name); do
   kubectl patch "$subnet" --type='json' -p '[{"op": "replace", "path": "/metadata/finalizers", "value": []}]'
+  kubectl delete "$subnet"
 done
 
 for vlan in $(kubectl get vlan -o name); do
@@ -13,31 +14,46 @@ for pn in $(kubectl get provider-network -o name); do
   kubectl delete $pn
 done
 
-sleep 3
+sleep 5
 
 # Delete Kube-OVN components
-kubectl delete cm ovn-config ovn-ic-config ovn-external-gw-config -n kube-system --ignore-not-found=true
-kubectl delete secret kube-ovn-tls -n kube-system --ignore-not-found=true
-kubectl delete sa ovn -n kube-system --ignore-not-found=true
-kubectl delete clusterrole system:ovn --ignore-not-found=true
-kubectl delete clusterrolebinding ovn --ignore-not-found=true
-kubectl delete svc ovn-nb ovn-sb ovn-northd kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system --ignore-not-found=true
-kubectl delete ds kube-ovn-cni -n kube-system --ignore-not-found=true
-kubectl delete deployment ovn-central kube-ovn-controller kube-ovn-monitor -n kube-system --ignore-not-found=true
-kubectl get pod --no-headers -n kube-system -lapp=ovs -o custom-columns=NAME:.metadata.name,STATUS:.status.phase,IP:.status.podIP | awk '{
-  if ($2 == "Running") {
-    system("kubectl exec -n kube-system "$1" -- bash /kube-ovn/uninstall.sh "$3)
-  }
-}'
-kubectl delete ds ovs-ovn kube-ovn-pinger -n kube-system --ignore-not-found=true
-kubectl delete crd --ignore-not-found=true \
-  ips.kubeovn.io \
-  subnets.kubeovn.io \
-  vpc-nat-gateways.kubeovn.io \
-  vpcs.kubeovn.io \
-  vlans.kubeovn.io \
-  provider-networks.kubeovn.io \
-  networks.kubeovn.io
+kubectl delete --ignore-not-found ds kube-ovn-pinger -n kube-system
+kubectl delete --ignore-not-found deploy kube-ovn-monitor -n kube-system
+kubectl delete --ignore-not-found cm ovn-config ovn-ic-config ovn-external-gw-config -n kube-system
+kubectl delete --ignore-not-found svc kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system
+kubectl delete --ignore-not-found ds kube-ovn-cni -n kube-system
+kubectl delete --ignore-not-found deploy kube-ovn-controller -n kube-system
+
+# ensure kube-ovn-cni has been deleted
+while :; do
+  if [ $(kubectl get pod --no-headers -n kube-system -l app=kube-ovn-cni | wc -l) -eq 0 ]; then
+    break
+  fi
+  sleep 5
+done
+
+for pod in $(kubectl get pod -n kube-system -l app=ovs -o 'jsonpath={.items[?(@.status.phase=="Running")].metadata.name}'); do
+  node=$(kubectl get pod -n kube-system $pod -o 'jsonpath={.spec.nodeName}')
+  nodeIPs=$(kubectl get node $node -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}' | sed 's/ /,/')
+  kubectl exec -n kube-system "$pod" -- bash /kube-ovn/uninstall.sh "$nodeIPs"
+done
+
+kubectl delete --ignore-not-found svc ovn-nb ovn-sb ovn-northd -n kube-system
+kubectl delete --ignore-not-found deploy ovn-central -n kube-system
+kubectl delete --ignore-not-found ds ovs-ovn -n kube-system
+kubectl delete --ignore-not-found secret kube-ovn-tls -n kube-system
+kubectl delete --ignore-not-found sa ovn -n kube-system
+kubectl delete --ignore-not-found clusterrole system:ovn
+kubectl delete --ignore-not-found clusterrolebinding ovn
+
+# delete CRD
+kubectl delete --ignore-not-found crd security-groups.kubeovn.io
+kubectl delete --ignore-not-found crd ips.kubeovn.io
+kubectl delete --ignore-not-found crd subnets.kubeovn.io
+kubectl delete --ignore-not-found crd vpc-nat-gateways.kubeovn.io
+kubectl delete --ignore-not-found crd vpcs.kubeovn.io
+kubectl delete --ignore-not-found crd vlans.kubeovn.io
+kubectl delete --ignore-not-found crd provider-networks.kubeovn.io
 
 # Remove annotations/labels in namespaces and nodes
 kubectl annotate no --all ovn.kubernetes.io/cidr-
@@ -49,6 +65,14 @@ kubectl annotate no --all ovn.kubernetes.io/port_name-
 kubectl annotate no --all ovn.kubernetes.io/allocated-
 kubectl label node --all kube-ovn/role-
 
+kubectl get no -o name | while read node; do
+  kubectl get "$node" -o 'go-template={{ range $k, $v := .metadata.labels }}{{ $k }}{{"\n"}}{{ end }}' | while read label; do
+    if echo "$label" | grep -qE '^(.+\.provider-network\.kubernetes\.io/(ready|mtu|interface|exclude))$'; then
+      kubectl label "$node" "$label-"
+    fi
+  done
+done
+
 kubectl annotate ns --all ovn.kubernetes.io/cidr-
 kubectl annotate ns --all ovn.kubernetes.io/exclude_ips-
 kubectl annotate ns --all ovn.kubernetes.io/gateway-
@@ -57,21 +81,26 @@ kubectl annotate ns --all ovn.kubernetes.io/private-
 kubectl annotate ns --all ovn.kubernetes.io/allow-
 kubectl annotate ns --all ovn.kubernetes.io/allocated-
 
-# Wait Pod Deletion
-sleep 5
+# ensure ovs-ovn has been deleted
+while :; do
+  sleep 5
+  if [ $(kubectl get pod --no-headers -n kube-system -l app=ovs | wc -l) -eq 0 ]; then
+    break
+  fi
+done
 
 # Remove annotations in all pods of all namespaces
 for ns in $(kubectl get ns -o name |cut -c 11-); do
   echo "annotating pods in  ns:$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/cidr- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/gateway- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/ip_address- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/logical_switch- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/mac_address- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/port_name- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/allocated- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/routed- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/vlan_id- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/network_type- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/provider_network- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/cidr- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/gateway- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/ip_address- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/logical_switch- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/mac_address- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/port_name- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/allocated- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/routed- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/vlan_id- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/network_type- -n "$ns"
+  kubectl annotate pod --all ovn.kubernetes.io/provider_network- -n "$ns"
 done
diff --git a/dist/images/uninstall.sh b/dist/images/uninstall.sh
@@ -2,6 +2,25 @@
 /usr/share/openvswitch/scripts/ovs-ctl stop
 ovs-dpctl del-dp ovs-system
 
+nodeIPv4=""
+nodeIPv6=""
+if [ -n "$1" ]; then
+    if [[ "$1" =~ .*,.* ]]; then
+        nodeIPv4=${1%%,*}
+        nodeIPv6=${1##*,}
+        if [[ "$nodeIPv4" =~ .*:.* ]]; then
+            nodeIPv4=${1##*,}
+            nodeIPv6=${1%%,*}
+        fi
+    else
+        if [[ "$1" =~ .*:.* ]]; then
+            nodeIPv6=$1
+        else
+            nodeIPv4=$1
+        fi
+    fi
+fi
+
 iptables -t nat -D POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
 iptables -t nat -D POSTROUTING -m set --match-set ovn40local-pod-ip-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
 iptables -t nat -D POSTROUTING -m mark --mark 0x40000/0x40000 -j MASQUERADE
@@ -14,12 +33,14 @@ iptables -t filter -D FORWARD -m set --match-set ovn40subnets dst -j ACCEPT
 iptables -t filter -D FORWARD -m set --match-set ovn40subnets src -j ACCEPT
 iptables -t filter -D FORWARD -m set --match-set ovn40services dst -j ACCEPT
 iptables -t filter -D FORWARD -m set --match-set ovn40services src -j ACCEPT
-iptables -D OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0
+iptables -t filter -D OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0
 
-if [ -n "$1" ]; then
-    iptables -t nat -D POSTROUTING ! -s "$1" -m set --match-set ovn40subnets dst -j MASQUERADE
+if [ -n "$nodeIPv4" ]; then
+    iptables -t nat -D POSTROUTING ! -s "$nodeIPv4" -m set --match-set ovn40subnets dst -j MASQUERADE
 fi
 
+sleep 1
+
 ipset destroy ovn40subnets-nat
 ipset destroy ovn40subnets
 ipset destroy ovn40local-pod-ip-nat
@@ -38,12 +59,14 @@ ip6tables -t filter -D FORWARD -m set --match-set ovn60subnets dst -j ACCEPT
 ip6tables -t filter -D FORWARD -m set --match-set ovn60subnets src -j ACCEPT
 ip6tables -t filter -D FORWARD -m set --match-set ovn60services dst -j ACCEPT
 ip6tables -t filter -D FORWARD -m set --match-set ovn60services src -j ACCEPT
-ip6tables -D OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0
+ip6tables -t filter -D OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0
 
-if [ -n "$1" ]; then
-    ip6tables -t nat -D POSTROUTING ! -s "$1" -m set --match-set ovn60subnets dst -j MASQUERADE
+if [ -n "$nodeIPv6" ]; then
+    ip6tables -t nat -D POSTROUTING ! -s "$nodeIPv6" -m set --match-set ovn60subnets dst -j MASQUERADE
 fi
 
+sleep 1
+
 ipset destroy ovn6subnets-nat
 ipset destroy ovn60subnets
 ipset destroy ovn60local-pod-ip-nat