Add kubectl ko log (#2451)

* add kubectl-ko log
kubeovn · Mar 15, 2023 · 0b8e534 · 0b8e534
1 parent c3620cd
commit 0b8e534
Show file tree

Hide file tree

Showing 4 changed files with 229 additions and 0 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -359,6 +359,10 @@ jobs:
           E2E_NETWORK_MODE: ${{ matrix.mode }}
         run: make k8s-conformance-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
   k8s-netpol-e2e:
     name: Kubernetes Network Policy E2E
     if: |
@@ -454,6 +458,10 @@ jobs:
         working-directory: ${{ env.E2E_DIR }}
         run: make k8s-netpol-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
   k8s-netpol-legacy-e2e:
     name: Kubernetes Network Policy Legacy E2E
     if: |
@@ -549,6 +557,10 @@ jobs:
         working-directory: ${{ env.E2E_DIR }}
         run: make k8s-netpol-legacy-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
   cyclonus-netpol-e2e:
     name: Cyclonus Network Policy E2E
     if: |
@@ -621,6 +633,10 @@ jobs:
         working-directory: ${{ env.E2E_DIR }}
         run: make cyclonus-netpol-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
   kube-ovn-conformance-e2e:
     name: Kube-OVN Conformance E2E
     needs:
@@ -710,6 +726,10 @@ jobs:
           E2E_IP_FAMILY: ${{ matrix.ip-family }}
         run: make kube-ovn-conformance-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
       - name: Cleanup
         run: sh dist/images/cleanup.sh
 
@@ -792,6 +812,10 @@ jobs:
         working-directory: ${{ env.E2E_DIR }}
         run: make kube-ovn-ic-conformance-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
   chart-installation-test:
     name: Chart Installation Test
     needs: build-kube-ovn
@@ -1026,6 +1050,7 @@ jobs:
         working-directory: ${{ env.E2E_DIR }}
         run: make kube-ovn-lb-svc-conformance-e2e
 
+
   kubevirt-e2e:
     name: Kubevirt vm E2E
     needs:
@@ -1186,6 +1211,10 @@ jobs:
         working-directory: ${{ env.E2E_DIR }}
         run: make kube-ovn-webhook-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
   installation-compatibility-test:
     name: Installation Compatibility Test
     needs: build-kube-ovn
@@ -1218,6 +1247,10 @@ jobs:
       - name: Install Kube-OVN
         run: make kind-install
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
       - name: Cleanup
         run: sh dist/images/cleanup.sh
 
@@ -1310,6 +1343,10 @@ jobs:
           E2E_CILIUM_CHAINING: "true"
         run: make k8s-conformance-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
       - name: Cleanup
         run: sh dist/images/cleanup.sh
 
@@ -1410,6 +1447,10 @@ jobs:
           E2E_IP_FAMILY: ${{ matrix.ip-family }}
         run: make kube-ovn-security-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
       - name: Cleanup
         run: sh dist/images/cleanup.sh
 
@@ -1565,5 +1606,9 @@ jobs:
         working-directory: ${{ env.E2E_DIR }}
         run: make kube-ovn-submariner-conformance-e2e
 
+      - name: kubectl ko log
+        if: failure() && steps.e2e.outcome != 'success'
+        run: make kubectl-ko-log
+
       - name: Cleanup
         run: sh dist/images/cleanup.sh
diff --git a/Makefile.e2e b/Makefile.e2e
@@ -179,3 +179,7 @@ kube-ovn-webhook-e2e:
 	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
 	ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \
 		--focus=CNI:Kube-OVN ./test/e2e/webhook/webhook.test
+
+.PHONY: kubectl-ko-log
+kubectl-ko-log:
+	kubectl ko log all all all
diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko
@@ -26,6 +26,15 @@ showHelp(){
   echo "  env-check    check the environment configuration"
   echo "  tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]    deploy kernel optimisation components to the system"
   echo "  reload    restart all kube-ovn components"
+  echo "  log {kube-ovn|ovn|ovs|linux|all} {all|sub_component...} [tail_lines|all]
+          kube-ovn with sub_components: kube-ovn-controller,kube-ovn-cni,kube-ovn-pinger,kube-ovn-monitor
+          ovn with sub_components: ovn-controller,ovn-northd,ovsdb-server-nb,ovsdb-server-sb
+          ovs with sub_components: ovs-vswitchd,ovsdb-server
+          linux with sub_components: dmesg,iptables,route,link,neigh,memory,top,sysctl,netstat
+          eg:   kubectl ko log all all all       --   show all component logs
+                kubectl ko log all               --   show all component logs with default tail_line=100
+                kubectl ko log kube-ovn all 1000 --   show kube-ovn all sub_component with tail_line=1000
+          "
 }
 
 # usage: ipv4_to_hex 192.168.0.1
@@ -940,6 +949,155 @@ env-check(){
   done
 }
 
+log_kube_ovn(){
+  sub_component_param=$1
+  tail_lines_param=$2
+  podNames=`kubectl get pod -n kube-system -l app=$sub_component_param -o 'jsonpath={.items[*].metadata.name}'`
+  for pod in $podNames; do
+    echo "************************************************"
+    echo " component $component sub_component $sub_component_param pod $pod log"
+    echo "************************************************"
+    kubectl logs "$pod" -n kube-system $( [ "$tail_lines" == "all" ] || echo "--tail=$tail_lines" ) 2>&1 || :
+  done
+}
+
+log_ovn_ovs(){
+  component_param=$1
+  sub_component_param=$2
+  tail_lines_param=$3
+  podNames=`kubectl get pod -n kube-system -l app=ovs -o 'jsonpath={.items[*].metadata.name}'`
+  for pod in $podNames; do
+    echo "************************************************"
+    echo " component $component_param $sub_component_param.log pod $pod log"
+    echo "************************************************"
+
+    if [[ "$tail_lines" == "all" ]]; then
+      kubectl exec $pod -n kube-system -- cat /var/log/$component_param/$sub_component_param.log 2>&1 || :
+    else
+      kubectl exec $pod -n kube-system -- tail -n $tail_lines_param /var/log/$component_param/$sub_component_param.log 2>&1 || :
+    fi
+  done
+}
+
+log_linux(){
+  sub_component_param=$1
+  tail_lines_param=$2
+  podNames=`kubectl get pod -n kube-system -l app=ovs -o 'jsonpath={.items[*].metadata.name}'`
+  for pod in $podNames; do
+    echo "************************************************"
+    echo " component $component sub_component $sub_component_param pod $pod log"
+    echo "************************************************"
+    if [[ "$sub_component_param" == "dmesg" ]]; then
+      kubectl exec $pod -n kube-system -- dmesg -T $( [ "$tail_lines" == "all" ] || echo "| tail -n $tail_lines_param" ) || :
+    elif [[ "$sub_component_param" == "iptables" ]]; then
+      echo "******************legacy filter ************************"
+      kubectl exec $pod -n kube-system -- iptables -S || :
+      echo "*********************nft filter ************************"
+      kubectl exec $pod -n kube-system -- /usr/sbin/iptables-nft -S || :
+
+      echo "****************** legacy nat ************************"
+      kubectl exec $pod -n kube-system -- iptables -S -t nat || :
+      echo "********************* nft nat ************************"
+      kubectl exec $pod -n kube-system -- /usr/sbin/iptables-nft -S -t nat || :
+
+    elif [[ "$sub_component_param" == "route" ]]; then
+      kubectl exec $pod -n kube-system -- ip route show || :
+    elif [[ "$sub_component_param" == "link" ]]; then
+      kubectl exec $pod -n kube-system -- ip -d link show || :
+    elif [[ "$sub_component_param" == "neigh" ]]; then
+      kubectl exec $pod -n kube-system -- ip n || :
+    elif [[ "$sub_component_param" == "memory" ]]; then
+      kubectl exec $pod -n kube-system -- free -m || :
+    elif [[ "$sub_component_param" == "top" ]]; then
+      kubectl exec $pod -n kube-system -- top -b -n 1 | head -n 20 | tail -n 10 || :
+    elif [[ "$sub_component_param" == "sysctl" ]]; then
+      kubectl exec $pod -n kube-system -- sysctl -a || :
+    elif [[ "$sub_component_param" == "netstat" ]]; then
+      kubectl exec $pod -n kube-system -- netstat -tunlp || :
+    fi
+  done
+}
+
+
+log(){
+  component="$1"
+  sub_component=${2:-all}
+  tail_lines=${3:-100}
+
+  components=("kube-ovn" "ovs" "ovn" "linux" "all")
+  kube_ovn_sub_components=("kube-ovn-controller" "kube-ovn-cni" "kube-ovn-pinger" "kube-ovn-monitor")
+  ovs_sub_components=("ovs-vswitchd" "ovsdb-server")
+  ovn_sub_components=("ovn-controller" "ovn-northd" "ovsdb-server-nb" "ovsdb-server-sb")
+  linux_sub_components=("dmesg" "iptables" "route" "link" "neigh" "memory" "top" "sysctl" "netstat")
+
+  if [[ ! " ${components[@]} " =~ " $component " ]]; then
+      echo "invalid component $component"
+      exit 1
+  fi
+
+  if [[ ! "$sub_component" == "all" && ! "$components" == "all" ]]; then
+    case $component in
+    "kube-ovn")
+        sub_components=("${kube_ovn_sub_components[@]}")
+        ;;
+    "ovs")
+        sub_components=("${ovs_sub_components[@]}")
+        ;;
+    "ovn")
+        sub_components=("${ovn_sub_components[@]}")
+        ;;
+    "linux")
+        sub_components=("${linux_sub_components[@]}")
+        ;;
+    esac
+
+    if [[ ! " ${sub_components[@]} " =~ " $sub_component " ]]; then
+      echo "component $component do not have sub component $sub_component"
+      exit 1
+    fi
+  fi
+
+  if [[ "$component" == "kube-ovn" || "$component" == "all" ]]; then
+    if [[ "$sub_component" == "all" ]]; then
+      for kube_ovn_sub_component in ${kube_ovn_sub_components[@]}; do
+        log_kube_ovn "$kube_ovn_sub_component" $tail_lines
+      done
+    else
+      log_kube_ovn "$sub_component" $tail_lines
+    fi
+  fi
+
+  if [[ "$component" == "ovn" || "$component" == "all" ]]; then
+    if [[ "$sub_component" == "all" ]]; then
+      for ovn_sub_component in ${ovn_sub_components[@]}; do
+        log_ovn_ovs "ovn" "$ovn_sub_component" $tail_lines
+      done
+    else
+      log_ovn_ovs "ovn" "$sub_component" $tail_lines
+    fi
+  fi
+
+  if [[ "$component" == "ovs" || "$component" == "all" ]]; then
+    if [[ "$sub_component" == "all" ]]; then
+      for ovs_sub_component in ${ovs_sub_components[@]}; do
+        log_ovn_ovs "openvswitch" "$ovs_sub_component" $tail_lines
+      done
+    else
+      log_ovn_ovs "openvswitch" "$sub_component" $tail_lines
+    fi
+  fi
+
+  if [[ "$component" == "linux" || "$component" == "all" ]]; then
+    if [[ "$sub_component" == "all" ]]; then
+      for linux_sub_component in ${linux_sub_components[@]}; do
+        log_linux "$linux_sub_component" $tail_lines
+      done
+    else
+      log_linux "$sub_component" $tail_lines
+    fi
+  fi
+}
+
 if [ $# -lt 1 ]; then
   showHelp
   exit 0
@@ -980,6 +1138,9 @@ case $subcommand in
   env-check)
     env-check
     ;;
+  log)
+    log "$@"
+    ;;
   *)
     showHelp
     exit 1

diff --git a/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go b/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go
@@ -142,4 +142,23 @@ var _ = framework.Describe("[group:kubectl-ko]", func() {
 		ginkgo.By("Deleting pod " + name)
 		podClient.DeleteSync(pod.Name)
 	})
+
+	framework.ConformanceIt(`should succeed to execute "kubectl ko log kube-ovn all"`, func() {
+		components := [...]string{"kube-ovn", "ovn", "ovs", "linux"}
+		for _, component := range components {
+			execOrDie(fmt.Sprintf("ko log %s all 10", component))
+		}
+
+		subComponentMap := make(map[string][]string)
+		subComponentMap["kube-ovn"] = []string{"kube-ovn-controller", "kube-ovn-cni", "kube-ovn-pinger", "kube-ovn-monitor"}
+		subComponentMap["ovs"] = []string{"ovs-vswitchd", "ovsdb-server"}
+		subComponentMap["ovn"] = []string{"ovn-controller", "ovn-northd", "ovsdb-server-nb", "ovsdb-server-sb"}
+		subComponentMap["linux"] = []string{"dmesg", "iptables", "route", "link", "neigh", "memory", "top"}
+
+		for component, subComponents := range subComponentMap {
+			for _, subComponent := range subComponents {
+				execOrDie(fmt.Sprintf("ko log %s %s 10", component, subComponent))
+			}
+		}
+	})
 })