fix: clean lost interface.

When reboot node, the lost interface and qos will prevent pod deletion and new pod creation.

cmd/daemon/cniserver.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bitbucket.org/mathildetech/kube-ovn/pkg/daemon"
+	"bitbucket.org/mathildetech/kube-ovn/pkg/ovs"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog"
 	"k8s.io/sample-controller/pkg/signals"
@@ -12,6 +13,7 @@ import (
 func main() {
 	klog.SetOutput(os.Stdout)
 	defer klog.Flush()
+	go gc()
 
 	config, err := daemon.ParseFlags()
 	if err != nil {
@@ -26,3 +28,10 @@ func main() {
 	go ctl.Run(stopCh)
 	daemon.RunServer(config)
 }
+
+func gc() {
+	for {
+		ovs.CleanLostInterface()
+		time.Sleep(60 * time.Second)
+	}
+}

pkg/controller/init.go

@@ -98,7 +98,6 @@ func InitLoadBalancer(config *Configuration) error {
 			return err
 		}
 	} else {
-		ovs.GlobalTcpLb = tcpLb
 		klog.Infof("tcp load balancer %s exists", tcpLb)
 	}
 
@@ -114,7 +113,6 @@ func InitLoadBalancer(config *Configuration) error {
 			return err
 		}
 	} else {
-		ovs.GlobalUdpLb = udpLb
 		klog.Infof("udp load balancer %s exists", udpLb)
 	}
 	return nil

pkg/daemon/ovs.go

@@ -67,18 +67,16 @@ func (csh CniServerHandler) configureNic(podName, podNamespace, netns, container
 }
 
 func (csh CniServerHandler) deleteNic(netns, podName, podNamespace, containerID string) error {
-	err := ovs.ClearPodBandwidth(podName, podNamespace)
-	if err != nil {
-		return err
-	}
-
 	hostNicName, _ := generateNicName(containerID)
 	// Remove ovs port
 	output, err := exec.Command("ovs-vsctl", "--if-exists", "--with-iface", "del-port", "br-int", hostNicName).CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("failed to delete ovs port %v, %s", err, output)
 	}
-
+	err = ovs.ClearPodBandwidth(podName, podNamespace)
+	if err != nil {
+		return err
+	}
 	hostLink, err := netlink.LinkByName(hostNicName)
 	if err != nil {
 		// If link already not exists, return quietly

pkg/ovs/ovn-nbctl.go

@@ -27,8 +27,6 @@ const (
 )
 
 var GlobalDnsTable string
-var GlobalTcpLb string
-var GlobalUdpLb string
 
 func (c Client) ovnCommand(arg ...string) (string, error) {
 	cmdArgs := []string{fmt.Sprintf("--db=%s", c.OvnNbAddress)}

pkg/ovs/ovs-vsctl.go

@@ -72,20 +72,6 @@ func ovsClear(table, record string, columns ...string) error {
 }
 
 func ClearPodBandwidth(podName, podNamespace string) error {
-	// interfaces will have the same name as ports
-	interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s.%s", podName, podNamespace))
-	if err != nil {
-		return err
-	}
-
-	// Clear the QoS for any ports of this sandbox
-	for _, ifName := range interfaceList {
-		if err = ovsClear("port", ifName, "qos"); err != nil {
-			return err
-		}
-	}
-
-	// Now that the QoS is unused remove it
 	qosList, err := ovsFind("qos", "_uuid", fmt.Sprintf(`external-ids:iface-id="%s.%s"`, podName, podNamespace))
 	if err != nil {
 		return err
@@ -151,3 +137,45 @@ func SetPodBandwidth(podName, podNamespace, ingress, egress string) error {
 	}
 	return nil
 }
+
+// When reboot node, the ovs internal interface will be deleted.
+// We need to clean up related ovs port, interface and qos
+func CleanLostInterface() {
+	// when interface error ofport will be -1
+	interfaceList, err := ovsFind("interface", "name,error", "ofport=-1")
+	if err != nil {
+		klog.Errorf("failed to list failed interface %v", err)
+		return
+	}
+	if len(interfaceList) > 0 {
+		klog.Infof("error interfaces:\n %v", interfaceList)
+	}
+
+	for _, intf := range interfaceList {
+		name, errText := strings.Trim(strings.Split(intf, "\n")[0], "\""), strings.Split(intf, "\n")[1]
+		if strings.Contains(errText, "No such device") {
+			qosList, err := ovsFind("port", "qos", fmt.Sprintf("name=%s", name))
+			if err != nil {
+				klog.Errorf("failed to find related port %v", err)
+				return
+			}
+			klog.Infof("delete lost port %s", name)
+			output, err := exec.Command("ovs-vsctl", "--if-exists", "--with-iface", "del-port", "br-int", name).CombinedOutput()
+			if err != nil {
+				klog.Errorf("failed to delete ovs port %v, %s", err, output)
+				return
+			}
+			for _, qos := range qosList {
+				qos = strings.TrimSpace(qos)
+				if qos != "" && qos != "[]" {
+					klog.Infof("delete lost qos %s", qos)
+					err = ovsDestroy("qos", qos)
+					if err != nil {
+						klog.Errorf("failed to delete qos %s, %v", qos, err)
+						return
+					}
+				}
+			}
+		}
+	}
+}