optimized initialization and gc for the chassis (#1511)

pkg/controller/controller.go

@@ -497,6 +497,10 @@ func (c *Controller) Run(stopCh <-chan struct{}) {
 		klog.Fatalf("failed to init ipam: %v", err)
 	}
 
+	if err := c.initNodeChassis(); err != nil {
+		klog.Errorf("failed to init node chassis: %v", err)
+	}
+
 	if err := c.initNodeRoutes(); err != nil {
 		klog.Fatalf("failed to initialize node routes: %v", err)
 	}

pkg/controller/gc.go

@@ -23,6 +23,7 @@ var lastNoPodLSP map[string]bool
 func (c *Controller) gc() error {
 	gcFunctions := []func() error{
 		c.gcNode,
+		c.gcChassis,
 		c.gcLogicalSwitch,
 		c.gcCustomLogicalRouter,
 		c.gcLogicalSwitchPort,
@@ -616,6 +617,35 @@ func (c *Controller) gcStaticRoute() error {
 	return nil
 }
 
+func (c *Controller) gcChassis() error {
+	klog.Infof("start to gc chassis")
+	chassises, err := c.ovnLegacyClient.GetAllChassis()
+	if err != nil {
+		klog.Errorf("failed to get all chassis, %v", err)
+	}
+	nodes, err := c.nodesLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list nodes, %v", err)
+		return err
+	}
+	for _, chassis := range chassises {
+		matched := true
+		for _, node := range nodes {
+			if chassis == node.Annotations[util.ChassisAnnotation] {
+				matched = false
+				break
+			}
+		}
+		if matched {
+			if err := c.ovnLegacyClient.DeleteChassisByName(chassis); err != nil {
+				klog.Errorf("failed to delete chassis %s %v", chassis, err)
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func (c *Controller) isOVNProvided(providerName string, pod *corev1.Pod) (bool, error) {
 	ls := pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, providerName)]
 	subnet, err := c.config.KubeOvnClient.KubeovnV1().Subnets().Get(context.Background(), ls, metav1.GetOptions{})

pkg/controller/init.go

@@ -764,3 +764,30 @@ func (c *Controller) initHtbQos() error {
 	}
 	return err
 }
+
+func (c *Controller) initNodeChassis() error {
+	nodes, err := c.nodesLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list nodes: %v", err)
+		return err
+	}
+
+	for _, node := range nodes {
+		chassisName := node.Annotations[util.ChassisAnnotation]
+		if chassisName != "" {
+			exist, err := c.ovnLegacyClient.ChassisExist(chassisName)
+			if err != nil {
+				klog.Errorf("failed to check chassis exist: %v", err)
+				return err
+			}
+			if exist {
+				err = c.ovnLegacyClient.InitChassisNodeTag(chassisName, node.Name)
+				if err != nil {
+					klog.Errorf("failed to set chassis nodeTag: %v", err)
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

pkg/controller/node.go

@@ -342,7 +342,7 @@ func (c *Controller) handleAddNode(key string) error {
 		return err
 	}
 
-	if err := c.RemoveRedundantChassis(node); err != nil {
+	if err := c.validateChassis(node); err != nil {
 		return err
 	}
 
@@ -445,7 +445,7 @@ func (c *Controller) handleDeleteNode(key string) error {
 		klog.Errorf("failed to delete node switch port node-%s: %v", key, err)
 		return err
 	}
-	if err := c.ovnLegacyClient.DeleteChassis(key); err != nil {
+	if err := c.ovnLegacyClient.DeleteChassisByNode(key); err != nil {
 		klog.Errorf("failed to delete chassis for node %s: %v", key, err)
 		return err
 	}
@@ -575,6 +575,12 @@ func (c *Controller) handleUpdateNode(key string) error {
 		return err
 	}
 
+	if node.Annotations[util.ChassisAnnotation] != "" {
+		if err = c.ovnLegacyClient.InitChassisNodeTag(node.Annotations[util.ChassisAnnotation], node.Name); err != nil {
+			klog.Errorf("failed to set chassis nodeTag for node '%s', %v", node.Name, err)
+			return err
+		}
+	}
 	if err := c.retryDelDupChassis(util.ChasRetryTime, util.ChasRetryIntev+2, c.checkChassisDupl, node); err != nil {
 		return err
 	}
@@ -835,7 +841,7 @@ func (c *Controller) checkChassisDupl(node *v1.Node) error {
 	}
 
 	klog.Errorf("duplicate chassis for node %s and new chassis %s", node.Name, chassisAdd)
-	if err := c.ovnLegacyClient.DeleteChassis(node.Name); err != nil {
+	if err := c.ovnLegacyClient.DeleteChassisByNode(node.Name); err != nil {
 		klog.Errorf("failed to delete chassis for node %s %v", node.Name, err)
 		return err
 	}
@@ -988,40 +994,35 @@ func (c *Controller) checkAndUpdateNodePortGroup() error {
 	return nil
 }
 
-func (c *Controller) RemoveRedundantChassis(node *v1.Node) error {
+func (c *Controller) validateChassis(node *v1.Node) error {
+	if node.Annotations[util.ChassisAnnotation] == "" {
+		return nil
+	}
 	chassisAdd, err := c.ovnLegacyClient.GetChassis(node.Name)
 	if err != nil {
 		klog.Errorf("failed to get node %s chassisID, %v", node.Name, err)
 		return err
 	}
-
-	if chassisAdd == "" && node.Annotations[util.ChassisAnnotation] != "" {
-		chassises, err := c.ovnLegacyClient.GetAllChassisHostname()
-		if err != nil {
-			klog.Errorf("failed to get all chassis, %v", err)
+	if node.Annotations[util.ChassisAnnotation] == chassisAdd {
+		if err = c.ovnLegacyClient.InitChassisNodeTag(chassisAdd, node.Name); err != nil {
+			return fmt.Errorf("failed to init chassis tag, %v", err)
 		}
-		nodes, err := c.nodesLister.List(labels.Everything())
-		if err != nil {
-			klog.Errorf("failed to list nodes, %v", err)
-			return err
+		return nil
+	}
+	if chassisAdd == "" {
+		// If no chassisID for this node is obtained, we need to perform GC in order for the chassis to be re-registered
+		if err = c.gcChassis(); err != nil {
+			return fmt.Errorf("failed to gc chassis, %v", err)
 		}
-		for _, chassis := range chassises {
-			matched := true
-			for _, node := range nodes {
-				if chassis == node.Name {
-					matched = false
-				}
-			}
-			if matched {
-				if err := c.ovnLegacyClient.DeleteChassis(chassis); err != nil {
-					klog.Errorf("failed to delete chassis for node %s %v", chassis, err)
-					return err
-				}
-			}
+	} else {
+		// When the ids are obtained but are inconsistent, it is usually because there are duplicate chassis records.
+		// All chassis related to Node need to be deleted so that the correct chassis can be registered again
+		if err := c.ovnLegacyClient.DeleteChassisByNode(node.Name); err != nil {
+			klog.Errorf("failed to delete chassis for node %s %v", node.Name, err)
+			return err
 		}
-		return errors.New("chassis reset, reboot ovs-ovn on this node: " + node.Name)
 	}
-	return nil
+	return errors.New("chassis reset, reboot ovs-ovn on this node: " + node.Name)
 }
 
 func (c *Controller) addNodeGwStaticRoute() error {

pkg/ovs/ovn-nbctl-legacy.go

@@ -53,6 +53,21 @@ func (c LegacyClient) ovnNbCommand(cmdArgs ...string) (string, error) {
 	return trimCommandOutput(raw), nil
 }
 
+func (c LegacyClient) GetVersion() (string, error) {
+	if c.Version != "" {
+		return c.Version, nil
+	}
+	output, err := c.ovnNbCommand("--version")
+	if err != nil {
+		return "", fmt.Errorf("failed to get version,%v", err)
+	}
+	lines := strings.Split(output, "\n")
+	if len(lines) > 0 {
+		c.Version = strings.Split(lines[0], " ")[1]
+	}
+	return c.Version, nil
+}
+
 func (c LegacyClient) SetAzName(azName string) error {
 	if _, err := c.ovnNbCommand("set", "NB_Global", ".", fmt.Sprintf("name=%s", azName)); err != nil {
 		return fmt.Errorf("failed to set az name, %v", err)

pkg/ovs/ovn-sbctl.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/kubeovn/kube-ovn/pkg/util"
 	"k8s.io/klog/v2"
 )
 
@@ -49,32 +50,74 @@ func (c LegacyClient) ovnSbCommand(cmdArgs ...string) (string, error) {
 	return trimCommandOutput(raw), nil
 }
 
-func (c LegacyClient) DeleteChassis(node string) error {
-	output, err := c.ovnSbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=name", "find", "chassis", fmt.Sprintf("hostname=%s", node))
+func (c LegacyClient) DeleteChassisByNode(node string) error {
+	output, err := c.ovnSbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=name", "find", "chassis", fmt.Sprintf("external_ids:node=%s", node))
 	if err != nil {
-		return fmt.Errorf("failed to find node chassis %s, %v", node, err)
+		return fmt.Errorf("failed to get node chassis %s, %v", node, err)
 	}
 	for _, chassis := range strings.Split(output, "\n") {
 		chassis = strings.TrimSpace(chassis)
 		if len(chassis) > 0 {
-			if _, err := c.ovnSbCommand("chassis-del", strings.TrimSpace(chassis), "--", "destroy", "chassis_private", strings.TrimSpace(chassis)); err != nil {
+			if err := c.DeleteChassisByName(chassis); err != nil {
 				return err
 			}
 		}
 	}
 	return nil
 }
 
+func (c LegacyClient) DeleteChassisByName(chassisName string) error {
+	ovnVersion, err := c.GetVersion()
+	if err != nil {
+		return fmt.Errorf("fieled to get ovn version, %v", err)
+	}
+
+	cmdArg := []string{"chassis-del", strings.TrimSpace(chassisName)}
+	if util.CompareVersion("20.09", ovnVersion) >= 0 {
+		cmdArg = append(cmdArg, "--", "destroy", "chassis_private", strings.TrimSpace(chassisName))
+	}
+	if _, err := c.ovnSbCommand(cmdArg...); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c LegacyClient) GetChassis(node string) (string, error) {
 	output, err := c.ovnSbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=name", "find", "chassis", fmt.Sprintf("hostname=%s", node))
 	if err != nil {
 		return "", fmt.Errorf("failed to find node chassis %s, %v", node, err)
 	}
+	if len(output) == 0 {
+		output, err = c.ovnSbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=name", "find", "chassis", fmt.Sprintf("external_ids:node=%s", node))
+		if err != nil {
+			return "", fmt.Errorf("failed to find node chassis %s, %v", node, err)
+		}
+	}
 	return strings.TrimSpace(output), nil
 }
 
-func (c LegacyClient) GetAllChassisHostname() ([]string, error) {
-	output, err := c.ovnSbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=hostname", "find", "chassis")
+func (c LegacyClient) ChassisExist(chassisName string) (bool, error) {
+	output, err := c.ovnSbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=name", "find", "chassis", fmt.Sprintf("name=%s", chassisName))
+	if err != nil {
+		return false, fmt.Errorf("failed to find node chassis %s, %v", chassisName, err)
+	}
+	if len(strings.Split(output, "\n")) == 0 {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (c LegacyClient) InitChassisNodeTag(chassisName string, nodeName string) error {
+	_, err := c.ovnSbCommand("set", "chassis", chassisName, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName), fmt.Sprintf("external_ids:node=%s", nodeName))
+	if err != nil {
+		return fmt.Errorf("failed to set chassis external_ids, %v", err)
+	}
+	return nil
+}
+
+// GetAllChassis get all chassis init by kube-ovn
+func (c LegacyClient) GetAllChassis() ([]string, error) {
+	output, err := c.ovnSbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=name", "find", "chassis", fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName))
 	if err != nil {
 		return nil, fmt.Errorf("failed to find node chassis, %v", err)
 	}

pkg/ovs/ovn.go

@@ -31,6 +31,7 @@ type LegacyClient struct {
 	NodeSwitch                    string
 	NodeSwitchCIDR                string
 	ExternalGatewayType           string
+	Version                       string
 }
 
 type OvnClient struct {

pkg/util/version.go

@@ -0,0 +1,31 @@
+package util
+
+import (
+	"strconv"
+	"strings"
+)
+
+// CompareVersion compare two version
+func CompareVersion(version1 string, version2 string) int {
+	versionA := strings.Split(version1, ".")
+	versionB := strings.Split(version2, ".")
+
+	for i := len(versionA); i < 4; i++ {
+		versionA = append(versionA, "0")
+	}
+	for i := len(versionB); i < 4; i++ {
+		versionB = append(versionB, "0")
+	}
+	for i := 0; i < 4; i++ {
+		version1, _ := strconv.Atoi(versionA[i])
+		version2, _ := strconv.Atoi(versionB[i])
+		if version1 == version2 {
+			continue
+		} else if version1 > version2 {
+			return 1
+		} else {
+			return -1
+		}
+	}
+	return 0
+}