diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index cba36e3b85c..efe1d2573fc 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -359,6 +359,10 @@ jobs: E2E_NETWORK_MODE: ${{ matrix.mode }} run: make k8s-conformance-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + k8s-netpol-e2e: name: Kubernetes Network Policy E2E if: | @@ -454,6 +458,10 @@ jobs: working-directory: ${{ env.E2E_DIR }} run: make k8s-netpol-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + k8s-netpol-legacy-e2e: name: Kubernetes Network Policy Legacy E2E if: | @@ -549,6 +557,10 @@ jobs: working-directory: ${{ env.E2E_DIR }} run: make k8s-netpol-legacy-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + cyclonus-netpol-e2e: name: Cyclonus Network Policy E2E if: | @@ -621,6 +633,10 @@ jobs: working-directory: ${{ env.E2E_DIR }} run: make cyclonus-netpol-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + kube-ovn-conformance-e2e: name: Kube-OVN Conformance E2E needs: @@ -710,6 +726,10 @@ jobs: E2E_IP_FAMILY: ${{ matrix.ip-family }} run: make kube-ovn-conformance-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + - name: Cleanup run: sh dist/images/cleanup.sh @@ -792,6 +812,10 @@ jobs: working-directory: ${{ env.E2E_DIR }} run: make kube-ovn-ic-conformance-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + chart-installation-test: name: Chart Installation Test needs: build-kube-ovn @@ -1026,6 +1050,7 @@ jobs: working-directory: ${{ env.E2E_DIR }} run: make kube-ovn-lb-svc-conformance-e2e + kubevirt-e2e: name: Kubevirt vm E2E needs: @@ -1186,6 +1211,10 @@ jobs: working-directory: ${{ env.E2E_DIR }} run: make kube-ovn-webhook-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + installation-compatibility-test: name: Installation Compatibility Test needs: build-kube-ovn @@ -1218,6 +1247,10 @@ jobs: - name: Install Kube-OVN run: make kind-install + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + - name: Cleanup run: sh dist/images/cleanup.sh @@ -1310,6 +1343,10 @@ jobs: E2E_CILIUM_CHAINING: "true" run: make k8s-conformance-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + - name: Cleanup run: sh dist/images/cleanup.sh @@ -1410,6 +1447,10 @@ jobs: E2E_IP_FAMILY: ${{ matrix.ip-family }} run: make kube-ovn-security-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + - name: Cleanup run: sh dist/images/cleanup.sh @@ -1565,5 +1606,9 @@ jobs: working-directory: ${{ env.E2E_DIR }} run: make kube-ovn-submariner-conformance-e2e + - name: kubectl ko log + if: failure() && steps.e2e.outcome != 'success' + run: make kubectl-ko-log + - name: Cleanup run: sh dist/images/cleanup.sh \ No newline at end of file diff --git a/Makefile.e2e b/Makefile.e2e index fdb62f484df..c03475a2316 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -179,3 +179,7 @@ kube-ovn-webhook-e2e: E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \ --focus=CNI:Kube-OVN ./test/e2e/webhook/webhook.test + +.PHONY: kubectl-ko-log +kubectl-ko-log: + kubectl ko log all all all diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index 093db4d6113..d7d70e77b5f 100755 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -26,6 +26,15 @@ showHelp(){ echo " env-check check the environment configuration" echo " tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version] deploy kernel optimisation components to the system" echo " reload restart all kube-ovn components" + echo " log {kube-ovn|ovn|ovs|linux|all} {all|sub_component...} [tail_lines|all] + kube-ovn with sub_components: kube-ovn-controller,kube-ovn-cni,kube-ovn-pinger,kube-ovn-monitor + ovn with sub_components: ovn-controller,ovn-northd,ovsdb-server-nb,ovsdb-server-sb + ovs with sub_components: ovs-vswitchd,ovsdb-server + linux with sub_components: dmesg,iptables,route,link,neigh,memory,top,sysctl,netstat + eg: kubectl ko log all all all -- show all component logs + kubectl ko log all -- show all component logs with default tail_line=100 + kubectl ko log kube-ovn all 1000 -- show kube-ovn all sub_component with tail_line=1000 + " } # usage: ipv4_to_hex 192.168.0.1 @@ -940,6 +949,155 @@ env-check(){ done } +log_kube_ovn(){ + sub_component_param=$1 + tail_lines_param=$2 + podNames=`kubectl get pod -n kube-system -l app=$sub_component_param -o 'jsonpath={.items[*].metadata.name}'` + for pod in $podNames; do + echo "************************************************" + echo " component $component sub_component $sub_component_param pod $pod log" + echo "************************************************" + kubectl logs "$pod" -n kube-system $( [ "$tail_lines" == "all" ] || echo "--tail=$tail_lines" ) 2>&1 || : + done +} + +log_ovn_ovs(){ + component_param=$1 + sub_component_param=$2 + tail_lines_param=$3 + podNames=`kubectl get pod -n kube-system -l app=ovs -o 'jsonpath={.items[*].metadata.name}'` + for pod in $podNames; do + echo "************************************************" + echo " component $component_param $sub_component_param.log pod $pod log" + echo "************************************************" + + if [[ "$tail_lines" == "all" ]]; then + kubectl exec $pod -n kube-system -- cat /var/log/$component_param/$sub_component_param.log 2>&1 || : + else + kubectl exec $pod -n kube-system -- tail -n $tail_lines_param /var/log/$component_param/$sub_component_param.log 2>&1 || : + fi + done +} + +log_linux(){ + sub_component_param=$1 + tail_lines_param=$2 + podNames=`kubectl get pod -n kube-system -l app=ovs -o 'jsonpath={.items[*].metadata.name}'` + for pod in $podNames; do + echo "************************************************" + echo " component $component sub_component $sub_component_param pod $pod log" + echo "************************************************" + if [[ "$sub_component_param" == "dmesg" ]]; then + kubectl exec $pod -n kube-system -- dmesg -T $( [ "$tail_lines" == "all" ] || echo "| tail -n $tail_lines_param" ) || : + elif [[ "$sub_component_param" == "iptables" ]]; then + echo "******************legacy filter ************************" + kubectl exec $pod -n kube-system -- iptables -S || : + echo "*********************nft filter ************************" + kubectl exec $pod -n kube-system -- /usr/sbin/iptables-nft -S || : + + echo "****************** legacy nat ************************" + kubectl exec $pod -n kube-system -- iptables -S -t nat || : + echo "********************* nft nat ************************" + kubectl exec $pod -n kube-system -- /usr/sbin/iptables-nft -S -t nat || : + + elif [[ "$sub_component_param" == "route" ]]; then + kubectl exec $pod -n kube-system -- ip route show || : + elif [[ "$sub_component_param" == "link" ]]; then + kubectl exec $pod -n kube-system -- ip -d link show || : + elif [[ "$sub_component_param" == "neigh" ]]; then + kubectl exec $pod -n kube-system -- ip n || : + elif [[ "$sub_component_param" == "memory" ]]; then + kubectl exec $pod -n kube-system -- free -m || : + elif [[ "$sub_component_param" == "top" ]]; then + kubectl exec $pod -n kube-system -- top -b -n 1 | head -n 20 | tail -n 10 || : + elif [[ "$sub_component_param" == "sysctl" ]]; then + kubectl exec $pod -n kube-system -- sysctl -a || : + elif [[ "$sub_component_param" == "netstat" ]]; then + kubectl exec $pod -n kube-system -- netstat -tunlp || : + fi + done +} + + +log(){ + component="$1" + sub_component=${2:-all} + tail_lines=${3:-100} + + components=("kube-ovn" "ovs" "ovn" "linux" "all") + kube_ovn_sub_components=("kube-ovn-controller" "kube-ovn-cni" "kube-ovn-pinger" "kube-ovn-monitor") + ovs_sub_components=("ovs-vswitchd" "ovsdb-server") + ovn_sub_components=("ovn-controller" "ovn-northd" "ovsdb-server-nb" "ovsdb-server-sb") + linux_sub_components=("dmesg" "iptables" "route" "link" "neigh" "memory" "top" "sysctl" "netstat") + + if [[ ! " ${components[@]} " =~ " $component " ]]; then + echo "invalid component $component" + exit 1 + fi + + if [[ ! "$sub_component" == "all" && ! "$components" == "all" ]]; then + case $component in + "kube-ovn") + sub_components=("${kube_ovn_sub_components[@]}") + ;; + "ovs") + sub_components=("${ovs_sub_components[@]}") + ;; + "ovn") + sub_components=("${ovn_sub_components[@]}") + ;; + "linux") + sub_components=("${linux_sub_components[@]}") + ;; + esac + + if [[ ! " ${sub_components[@]} " =~ " $sub_component " ]]; then + echo "component $component do not have sub component $sub_component" + exit 1 + fi + fi + + if [[ "$component" == "kube-ovn" || "$component" == "all" ]]; then + if [[ "$sub_component" == "all" ]]; then + for kube_ovn_sub_component in ${kube_ovn_sub_components[@]}; do + log_kube_ovn "$kube_ovn_sub_component" $tail_lines + done + else + log_kube_ovn "$sub_component" $tail_lines + fi + fi + + if [[ "$component" == "ovn" || "$component" == "all" ]]; then + if [[ "$sub_component" == "all" ]]; then + for ovn_sub_component in ${ovn_sub_components[@]}; do + log_ovn_ovs "ovn" "$ovn_sub_component" $tail_lines + done + else + log_ovn_ovs "ovn" "$sub_component" $tail_lines + fi + fi + + if [[ "$component" == "ovs" || "$component" == "all" ]]; then + if [[ "$sub_component" == "all" ]]; then + for ovs_sub_component in ${ovs_sub_components[@]}; do + log_ovn_ovs "openvswitch" "$ovs_sub_component" $tail_lines + done + else + log_ovn_ovs "openvswitch" "$sub_component" $tail_lines + fi + fi + + if [[ "$component" == "linux" || "$component" == "all" ]]; then + if [[ "$sub_component" == "all" ]]; then + for linux_sub_component in ${linux_sub_components[@]}; do + log_linux "$linux_sub_component" $tail_lines + done + else + log_linux "$sub_component" $tail_lines + fi + fi +} + if [ $# -lt 1 ]; then showHelp exit 0 @@ -980,6 +1138,9 @@ case $subcommand in env-check) env-check ;; + log) + log "$@" + ;; *) showHelp exit 1 diff --git a/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go b/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go index f306caa44d1..ab67a34103a 100644 --- a/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go +++ b/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go @@ -142,4 +142,23 @@ var _ = framework.Describe("[group:kubectl-ko]", func() { ginkgo.By("Deleting pod " + name) podClient.DeleteSync(pod.Name) }) + + framework.ConformanceIt(`should succeed to execute "kubectl ko log kube-ovn all"`, func() { + components := [...]string{"kube-ovn", "ovn", "ovs", "linux"} + for _, component := range components { + execOrDie(fmt.Sprintf("ko log %s all 10", component)) + } + + subComponentMap := make(map[string][]string) + subComponentMap["kube-ovn"] = []string{"kube-ovn-controller", "kube-ovn-cni", "kube-ovn-pinger", "kube-ovn-monitor"} + subComponentMap["ovs"] = []string{"ovs-vswitchd", "ovsdb-server"} + subComponentMap["ovn"] = []string{"ovn-controller", "ovn-northd", "ovsdb-server-nb", "ovsdb-server-sb"} + subComponentMap["linux"] = []string{"dmesg", "iptables", "route", "link", "neigh", "memory", "top"} + + for component, subComponents := range subComponentMap { + for _, subComponent := range subComponents { + execOrDie(fmt.Sprintf("ko log %s %s 10", component, subComponent)) + } + } + }) })