Fix kubectl ko log loss when restart deployment or ds (#2531)

* fix loss log when restart component
* 1. add kube-ovn-monitor log to /var/log/kube-ovn/
2. get kube-ovn-cni log on each node

dist/images/install.sh

@@ -3501,6 +3501,11 @@ spec:
           image: "$REGISTRY/kube-ovn:$VERSION"
           imagePullPolicy: $IMAGE_PULL_POLICY
           command: ["/kube-ovn/start-ovn-monitor.sh"]
+          args:
+          - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --log_file_max_size=0
           securityContext:
             runAsUser: 0
             privileged: false
@@ -3541,6 +3546,8 @@ spec:
               name: localtime
             - mountPath: /var/run/tls
               name: kube-ovn-tls
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
           readinessProbe:
             exec:
               command:
@@ -3586,6 +3593,9 @@ spec:
           secret:
             optional: true
             secretName: kube-ovn-tls
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
 ---
 kind: Service
 apiVersion: v1

dist/images/kubectl-ko

@@ -950,14 +950,20 @@ env-check(){
 }
 
 log_kube_ovn(){
-  sub_component_param=$1
-  tail_lines_param=$2
-  podNames=`kubectl get pod -n kube-system -l app=$sub_component_param -o 'jsonpath={.items[*].metadata.name}'`
+  component_param=$1
+  sub_component_param=$2
+  tail_lines_param=$3
+  podNames=`kubectl get pod -n kube-system -l app=kube-ovn-cni -o 'jsonpath={.items[*].metadata.name}'`
   for pod in $podNames; do
+    nodeName=$(kubectl get pod "$pod" -n kube-system -o jsonpath={.spec.nodeName})
     echo "************************************************"
-    echo " component $component sub_component $sub_component_param pod $pod log"
+    echo " component $component sub_component $sub_component_param on node $nodeName pod $pod log "
     echo "************************************************"
-    kubectl logs "$pod" -n kube-system $( [ "$tail_lines" == "all" ] || echo "--tail=$tail_lines" ) 2>&1 || :
+    if [[ "$tail_lines_param" == "all" ]]; then
+      kubectl exec $pod -n kube-system -c cni-server -- cat /var/log/$component_param/$sub_component_param.log 2>&1 || :
+    else
+      kubectl exec $pod -n kube-system -c cni-server -- tail -n $tail_lines_param /var/log/$component_param/$sub_component_param.log 2>&1 || :
+    fi
   done
 }
 
@@ -971,7 +977,7 @@ log_ovn_ovs(){
     echo " component $component_param $sub_component_param.log pod $pod log"
     echo "************************************************"
 
-    if [[ "$tail_lines" == "all" ]]; then
+    if [[ "$tail_lines_param" == "all" ]]; then
       kubectl exec $pod -n kube-system -- cat /var/log/$component_param/$sub_component_param.log 2>&1 || :
     else
       kubectl exec $pod -n kube-system -- tail -n $tail_lines_param /var/log/$component_param/$sub_component_param.log 2>&1 || :
@@ -988,7 +994,7 @@ log_linux(){
     echo " component $component sub_component $sub_component_param pod $pod log"
     echo "************************************************"
     if [[ "$sub_component_param" == "dmesg" ]]; then
-      kubectl exec $pod -n kube-system -- dmesg -T $( [ "$tail_lines" == "all" ] || echo "| tail -n $tail_lines_param" ) || :
+      kubectl exec $pod -n kube-system -- dmesg -T $( [ "$tail_lines_param" == "all" ] || echo "| tail -n $tail_lines_param" ) || :
     elif [[ "$sub_component_param" == "iptables" ]]; then
       echo "******************legacy filter ************************"
       kubectl exec $pod -n kube-system -- iptables -S || :
@@ -1060,10 +1066,10 @@ log(){
   if [[ "$component" == "kube-ovn" || "$component" == "all" ]]; then
     if [[ "$sub_component" == "all" ]]; then
       for kube_ovn_sub_component in ${kube_ovn_sub_components[@]}; do
-        log_kube_ovn "$kube_ovn_sub_component" $tail_lines
+        log_kube_ovn "kube-ovn" "$kube_ovn_sub_component" $tail_lines
       done
     else
-      log_kube_ovn "$sub_component" $tail_lines
+      log_kube_ovn "kube-ovn" "$sub_component" $tail_lines
     fi
   fi
 

kubeovn-helm/templates/monitor-deploy.yaml

@@ -43,6 +43,11 @@ spec:
           image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           command: ["/kube-ovn/start-ovn-monitor.sh"]
+          args:
+          - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --log_file_max_size=0
           securityContext:
             runAsUser: 0
             privileged: false
@@ -83,6 +88,8 @@ spec:
               name: localtime
             - mountPath: /var/run/tls
               name: kube-ovn-tls
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
           readinessProbe:
             exec:
               command:
@@ -128,3 +135,6 @@ spec:
           secret:
             optional: true
             secretName: kube-ovn-tls
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn

pkg/ovnmonitor/config.go

@@ -1,8 +1,12 @@
 package ovnmonitor
 
 import (
+	"flag"
+
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
+
+	"github.com/kubeovn/kube-ovn/pkg/util"
 )
 
 // Configuration contains parameters information.
@@ -86,6 +90,22 @@ func ParseFlags() (*Configuration, error) {
 		argServiceNorthdFilePidPath   = pflag.String("service.ovn.northd.file.pid.path", "/var/run/ovn/ovn-northd.pid", "OVN northd daemon process id file.")
 	)
 
+	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
+	klog.InitFlags(klogFlags)
+
+	// Sync the glog and klog flags.
+	pflag.CommandLine.VisitAll(func(f1 *pflag.Flag) {
+		f2 := klogFlags.Lookup(f1.Name)
+		if f2 != nil {
+			value := f1.Value.String()
+			if err := f2.Value.Set(value); err != nil {
+				util.LogFatalAndExit(err, "failed to set flag")
+			}
+		}
+	})
+
+	pflag.CommandLine.AddGoFlagSet(klogFlags)
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 	pflag.Parse()
 
 	config := &Configuration{

test/e2e/framework/deployment.go

@@ -120,13 +120,7 @@ func MakeDeployment(name string, replicas int32, podLabels, podAnnotations map[s
 	return deploy
 }
 
-func RestartSystemDeployment(name string, enableLog bool) {
-	if enableLog {
-		output, err := exec.Command("bash", "-c", fmt.Sprintf("kubectl ko log kube-ovn %s all", name)).CombinedOutput()
-		framework.ExpectNoError(err, fmt.Sprintf("dump all %s log failed", name))
-		framework.Logf(string(output))
-	}
-
+func RestartSystemDeployment(name string) {
 	restartCmd := fmt.Sprintf("kubectl rollout restart deployment %s -n kube-system", name)
 	_, err := exec.Command("bash", "-c", restartCmd).CombinedOutput()
 	framework.ExpectNoError(err, fmt.Sprintf("restart %s failed", name))

test/e2e/kube-ovn/underlay/underlay.go

@@ -544,7 +544,7 @@ var _ = framework.SerialDescribe("[group:underlay]", func() {
 		checkU2OItems(true, subnet, underlayPod, overlayPod)
 
 		ginkgo.By("step4: check if kube-ovn-controller restart")
-		framework.RestartSystemDeployment("kube-ovn-controller", true)
+		framework.RestartSystemDeployment("kube-ovn-controller")
 		checkU2OItems(true, subnet, underlayPod, overlayPod)
 
 		ginkgo.By("step5: Disable u2o check after restart kube-controller")

yamls/kube-ovn-dual-stack.yaml

@@ -420,6 +420,11 @@ spec:
           image: "kubeovn/kube-ovn:v1.10.0"
           imagePullPolicy: IfNotPresent
           command: ["/kube-ovn/start-ovn-monitor.sh"]
+          args:
+            - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
+            - --logtostderr=false
+            - --alsologtostderr=true
+            - --log_file_max_size=0
           securityContext:
             runAsUser: 0
             privileged: false
@@ -458,6 +463,8 @@ spec:
               name: localtime
             - mountPath: /var/run/tls
               name: kube-ovn-tls
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
           readinessProbe:
             exec:
               command:
@@ -503,6 +510,9 @@ spec:
           secret:
             optional: true
             secretName: kube-ovn-tls
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
 ---
 kind: Service
 apiVersion: v1

yamls/kube-ovn-ipv6.yaml

@@ -392,6 +392,11 @@ spec:
           image: "kubeovn/kube-ovn:v1.10.0"
           imagePullPolicy: IfNotPresent
           command: ["/kube-ovn/start-ovn-monitor.sh"]
+          args:
+          - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --log_file_max_size=0
           securityContext:
             runAsUser: 0
             privileged: false
@@ -430,6 +435,8 @@ spec:
               name: localtime
             - mountPath: /var/run/tls
               name: kube-ovn-tls
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
           readinessProbe:
             exec:
               command:
@@ -475,6 +482,9 @@ spec:
           secret:
             optional: true
             secretName: kube-ovn-tls
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
 ---
 kind: Service
 apiVersion: v1

yamls/kube-ovn.yaml

@@ -442,6 +442,11 @@ spec:
           image: "kubeovn/kube-ovn:v1.10.0"
           imagePullPolicy: IfNotPresent
           command: ["/kube-ovn/start-ovn-monitor.sh"]
+          args:
+          - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --log_file_max_size=0
           securityContext:
             runAsUser: 0
             privileged: false
@@ -480,6 +485,8 @@ spec:
               name: localtime
             - mountPath: /var/run/tls
               name: kube-ovn-tls
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
           readinessProbe:
             exec:
               command:
@@ -525,6 +532,9 @@ spec:
           secret:
             optional: true
             secretName: kube-ovn-tls
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
 ---
 kind: Service
 apiVersion: v1