fix: delay mv cni conf to when cniserver is ready

kubeovn · Sep 21, 2020 · f1c2f99 · f1c2f99
1 parent cfafbac
commit f1c2f99
Show file tree

Hide file tree

Showing 8 changed files with 82 additions and 56 deletions.
diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -3,13 +3,14 @@ package main
 import (
 	"fmt"
 	"github.com/alauda/kube-ovn/pkg/util"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"io/ioutil"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"net/http"
 	_ "net/http/pprof" // #nosec
 
 	kubeovninformer "github.com/alauda/kube-ovn/pkg/client/informers/externalversions"
 	"github.com/alauda/kube-ovn/pkg/daemon"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog"
 	"k8s.io/sample-controller/pkg/signals"
@@ -51,9 +52,18 @@ func main() {
 	kubeInformerFactory.Start(stopCh)
 	kubeovnInformerFactory.Start(stopCh)
 	go ctl.Run(stopCh)
-	go func() {
-		http.Handle("/metrics", promhttp.Handler())
-		klog.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", config.PprofPort), nil))
-	}()
-	daemon.RunServer(config)
+	go daemon.RunServer(config)
+	if err := mvCNIConf(); err != nil {
+		klog.Fatalf("failed to mv cni conf, %v", err)
+	}
+	http.Handle("/metrics", promhttp.Handler())
+	klog.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", config.PprofPort), nil))
+}
+
+func mvCNIConf() error {
+	data, err := ioutil.ReadFile("/kube-ovn/01-kube-ovn.conflist")
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile("/etc/cni/net.d/01-kube-ovn.conflist", data, 0444)
 }
diff --git a/dist/images/install-cni.sh b/dist/images/install-cni.sh
@@ -19,16 +19,13 @@ if [[ -f "/proc/sys/net/ipv4/conf/all/rp_filter" ]];
 fi
 
 exit_with_error(){
-  echo $1
+  echo "$1"
   exit 1
 }
 
 CNI_BIN_SRC=/kube-ovn/kube-ovn
 CNI_BIN_DST=/opt/cni/bin/kube-ovn
 
-CNI_CONF_SRC=/kube-ovn/01-kube-ovn.conflist
-CNI_CONF_DST=/etc/cni/net.d/01-kube-ovn.conflist
-
 LOOPBACK_BIN_SRC=/loopback
 LOOPBACK_BIN_DST=/opt/cni/bin/loopback
 
@@ -38,4 +35,3 @@ PORTMAP_BIN_DST=/opt/cni/bin/portmap
 yes | cp -f $LOOPBACK_BIN_SRC $LOOPBACK_BIN_DST || exit_with_error "Failed to copy $LOOPBACK_BIN_SRC to $LOOPBACK_BIN_DST"
 yes | cp -f $PORTMAP_BIN_SRC $PORTMAP_BIN_DST || exit_with_error "Failed to copy $PORTMAP_BIN_SRC to $PORTMAP_BIN_DST"
 yes | cp -f $CNI_BIN_SRC $CNI_BIN_DST || exit_with_error "Failed to copy $CNI_BIN_SRC to $CNI_BIN_DST"
-yes | cp -f $CNI_CONF_SRC $CNI_CONF_DST || exit_with_error "Failed to copy $CNI_CONF_SRC to $CNI_CONF_DST"
diff --git a/dist/images/install-pre-1.16.sh b/dist/images/install-pre-1.16.sh
@@ -1130,8 +1130,6 @@ spec:
           runAsUser: 0
           privileged: true
         volumeMounts:
-          - mountPath: /etc/cni/net.d
-            name: cni-conf
           - mountPath: /opt/cni/bin
             name: cni-bin
       containers:
@@ -1161,6 +1159,8 @@ spec:
               fieldRef:
                 fieldPath: spec.nodeName
         volumeMounts:
+          - mountPath: /etc/cni/net.d
+            name: cni-conf
           - mountPath: /run/openvswitch
             name: host-run-ovs
           - mountPath: /run/ovn
@@ -1355,6 +1355,7 @@ EOF
 
 kubectl apply -f kube-ovn.yaml
 kubectl rollout status deployment/kube-ovn-controller -n ${NAMESPACE}
+kubectl rollout status daemonset/kube-ovn-cni -n ${NAMESPACE}
 echo "-------------------------------"
 echo ""
 

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -1247,8 +1247,6 @@ spec:
           runAsUser: 0
           privileged: true
         volumeMounts:
-          - mountPath: /etc/cni/net.d
-            name: cni-conf
           - mountPath: /opt/cni/bin
             name: cni-bin
       containers:
@@ -1278,6 +1276,8 @@ spec:
               fieldRef:
                 fieldPath: spec.nodeName
         volumeMounts:
+          - mountPath: /etc/cni/net.d
+            name: cni-conf
           - mountPath: /run/openvswitch
             name: host-run-ovs
           - mountPath: /run/ovn
@@ -1472,6 +1472,7 @@ EOF
 
 kubectl apply -f kube-ovn.yaml
 kubectl rollout status deployment/kube-ovn-controller -n ${NAMESPACE}
+kubectl rollout status daemonset/kube-ovn-cni -n ${NAMESPACE}
 echo "-------------------------------"
 echo ""
 

diff --git a/dist/images/start-cniserver.sh b/dist/images/start-cniserver.sh
@@ -4,6 +4,12 @@ set -euo pipefail
 CNI_SOCK=/run/openvswitch/kube-ovn-daemon.sock
 OVS_SOCK=/run/openvswitch/db.sock
 
+function quit {
+    rm -rf CNI_CONF
+    exit 0
+}
+trap quit EXIT
+
 if [[ -e "$CNI_SOCK" ]]
 then
     echo "previous socket exists, remove and continue"
@@ -12,7 +18,7 @@ fi
 
 while true
 do
-  sleep 5
+  sleep 1
   if [[ -e "$OVS_SOCK" ]]
   then
     break
@@ -21,7 +27,8 @@ do
   fi
 done
 
-kubectl rollout status deployment/kube-ovn-controller -n $(cat /run/secrets/kubernetes.io/serviceaccount/namespace)
-sleep 5
+# wait kube-ovn-controller ready
+kubectl rollout status deployment/kube-ovn-controller -n "$(cat /run/secrets/kubernetes.io/serviceaccount/namespace)"
+sleep 1
 
-./kube-ovn-daemon --ovs-socket=${OVS_SOCK} --bind-socket=${CNI_SOCK} $@
+./kube-ovn-daemon --ovs-socket=${OVS_SOCK} --bind-socket=${CNI_SOCK} "$@"
diff --git a/yamls/kube-ovn-ipv6.yaml b/yamls/kube-ovn-ipv6.yaml
@@ -115,10 +115,8 @@ spec:
             runAsUser: 0
             privileged: true
           volumeMounts:
-            - mountPath: /etc/cni/net.d
-              name: cni-conf
-            - mountPath: /opt/cni/bin
-              name: cni-bin
+          - mountPath: /opt/cni/bin
+            name: cni-bin
       containers:
         - name: cni-server
           image: "kubeovn/kube-ovn:v1.4.0"
@@ -140,8 +138,15 @@ spec:
                 fieldRef:
                   fieldPath: spec.nodeName
           volumeMounts:
-            - mountPath: /run/openvswitch
-              name: host-run-ovs
+          - mountPath: /etc/cni/net.d
+            name: cni-conf
+          - mountPath: /run/openvswitch
+            name: host-run-ovs
+          - mountPath: /run/ovn
+            name: host-run-ovn
+          - mountPath: /var/run/netns
+            name: host-ns
+            mountPropagation: HostToContainer
           readinessProbe:
             exec:
               command:
@@ -165,15 +170,21 @@ spec:
       nodeSelector:
         beta.kubernetes.io/os: "linux"
       volumes:
-        - name: host-run-ovs
-          hostPath:
-            path: /run/openvswitch
-        - name: cni-conf
-          hostPath:
-            path: /etc/cni/net.d
-        - name: cni-bin
-          hostPath:
-            path: /opt/cni/bin
+      - name: host-run-ovs
+        hostPath:
+          path: /run/openvswitch
+      - name: host-run-ovn
+        hostPath:
+          path: /run/ovn
+      - name: cni-conf
+        hostPath:
+          path: /etc/cni/net.d
+      - name: cni-bin
+        hostPath:
+          path: /opt/cni/bin
+      - name: host-ns
+        hostPath:
+          path: /var/run/netns
 
 ---
 kind: DaemonSet

diff --git a/yamls/kube-ovn-pre17.yaml b/yamls/kube-ovn-pre17.yaml
@@ -113,10 +113,8 @@ spec:
           runAsUser: 0
           privileged: true
         volumeMounts:
-          - mountPath: /etc/cni/net.d
-            name: cni-conf
-          - mountPath: /opt/cni/bin
-            name: cni-bin
+        - mountPath: /opt/cni/bin
+          name: cni-bin
       containers:
       - name: cni-server
         image: "kubeovn/kube-ovn:v1.4.0"
@@ -139,13 +137,15 @@ spec:
               fieldRef:
                 fieldPath: spec.nodeName
         volumeMounts:
-          - mountPath: /run/openvswitch
-            name: host-run-ovs
-          - mountPath: /run/ovn
-            name: host-run-ovn
-          - mountPath: /var/run/netns
-            name: host-ns
-            mountPropagation: HostToContainer
+        - mountPath: /etc/cni/net.d
+          name: cni-conf
+        - mountPath: /run/openvswitch
+          name: host-run-ovs
+        - mountPath: /run/ovn
+          name: host-run-ovn
+        - mountPath: /var/run/netns
+          name: host-ns
+          mountPropagation: HostToContainer
         readinessProbe:
           exec:
             command:

diff --git a/yamls/kube-ovn.yaml b/yamls/kube-ovn.yaml
@@ -117,10 +117,8 @@ spec:
           runAsUser: 0
           privileged: true
         volumeMounts:
-          - mountPath: /etc/cni/net.d
-            name: cni-conf
-          - mountPath: /opt/cni/bin
-            name: cni-bin
+        - mountPath: /opt/cni/bin
+          name: cni-bin
       containers:
       - name: cni-server
         image: "kubeovn/kube-ovn:v1.4.0"
@@ -143,13 +141,15 @@ spec:
               fieldRef:
                 fieldPath: spec.nodeName
         volumeMounts:
-          - mountPath: /run/openvswitch
-            name: host-run-ovs
-          - mountPath: /run/ovn
-            name: host-run-ovn
-          - mountPath: /var/run/netns
-            name: host-ns
-            mountPropagation: HostToContainer
+        - mountPath: /etc/cni/net.d
+          name: cni-conf
+        - mountPath: /run/openvswitch
+          name: host-run-ovs
+        - mountPath: /run/ovn
+          name: host-run-ovn
+        - mountPath: /var/run/netns
+          name: host-ns
+          mountPropagation: HostToContainer
         readinessProbe:
           exec:
             command: