fix chassis check for node

cmd/daemon/cniserver.go

@@ -1,10 +1,16 @@
 package daemon
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"k8s.io/apimachinery/pkg/types"
 	"net/http"
 	_ "net/http/pprof" // #nosec
+	"os"
+	"strings"
+	"time"
 
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
@@ -24,6 +30,11 @@ func CmdMain() {
 	klog.Infof(versions.String())
 	daemon.InitMetrics()
 	config, err := daemon.ParseFlags()
+
+	if err := Retry(util.ChasRetryTime, util.ChasRetryIntev, initChassisAnno, config); err != nil {
+		klog.Fatalf("failed to annotate chassis id, %v", err)
+	}
+
 	if err != nil {
 		klog.Fatalf("parse config failed %v", err)
 	}
@@ -79,3 +90,56 @@ func mvCNIConf() error {
 	}
 	return ioutil.WriteFile("/etc/cni/net.d/01-kube-ovn.conflist", data, 0444)
 }
+
+func Retry(attempts int, sleep int, f func(configuration *daemon.Configuration) error, ctrl *daemon.Configuration) (err error) {
+	for i := 0; ; i++ {
+		err = f(ctrl)
+		if err == nil {
+			return
+		}
+		if i >= (attempts - 1) {
+			break
+		}
+		time.Sleep(time.Duration(sleep) * time.Second)
+	}
+	return err
+}
+
+func initChassisAnno(cfg *daemon.Configuration) error {
+
+	chassisID, err := ioutil.ReadFile(util.ChassisLoc)
+	if err != nil {
+		klog.Errorf("read chassis file failed, %v", err)
+		return err
+	}
+
+	hostname := os.Getenv(util.HostnameEnv)
+	if hostname == "" {
+		klog.Errorf("env %s does not exist", util.HostnameEnv)
+		return err
+	}
+	node, err := cfg.KubeClient.CoreV1().Nodes().Get(context.Background(), hostname, v1.GetOptions{})
+	if err != nil {
+		klog.Errorf("failed to get node %s %v", hostname, err)
+		return err
+	}
+
+	chassistr := string(chassisID)
+	node.Annotations[util.ChassisAnnotation] = strings.Replace(chassistr, "\n", "", -1)
+	patchPayloadTemplate :=
+		`[{
+        "op": "%s",
+        "path": "/metadata/annotations",
+        "value": %s
+    }]`
+	op := "add"
+	raw, _ := json.Marshal(node.Annotations)
+	patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
+	_, err = cfg.KubeClient.CoreV1().Nodes().Patch(context.Background(), hostname, types.JSONPatchType, []byte(patchPayload), v1.PatchOptions{}, "")
+	if err != nil {
+		klog.Errorf("patch node %s failed %v", hostname, err)
+		return err
+	}
+	klog.Infof("finish adding chassis annotation")
+	return nil
+}

dist/images/install.sh

@@ -1783,6 +1783,8 @@ spec:
               fieldRef:
                 fieldPath: spec.nodeName
         volumeMounts:
+          - mountPath: /etc/openvswitch
+            name: systemid
           - mountPath: /etc/cni/net.d
             name: cni-conf
           - mountPath: /run/openvswitch
@@ -1822,6 +1824,9 @@ spec:
       nodeSelector:
         kubernetes.io/os: "linux"
       volumes:
+        - name: systemid
+          hostPath:
+            path: /etc/origin/openvswitch
         - name: host-run-ovs
           hostPath:
             path: /run/openvswitch

pkg/controller/node.go

@@ -54,7 +54,8 @@ func (c *Controller) enqueueUpdateNode(oldObj, newObj interface{}) {
 	oldNode := oldObj.(*v1.Node)
 	newNode := newObj.(*v1.Node)
 
-	if nodeReady(oldNode) != nodeReady(newNode) {
+	if nodeReady(oldNode) != nodeReady(newNode) ||
+		oldNode.Annotations[util.ChassisAnnotation] != newNode.Annotations[util.ChassisAnnotation] {
 		var key string
 		var err error
 		if key, err = cache.MetaNamespaceKeyFunc(newObj); err != nil {
@@ -203,6 +204,10 @@ func (c *Controller) handleAddNode(key string) error {
 		return err
 	}
 
+	if err := c.checkChassisDupl(node); err != nil {
+		return err
+	}
+
 	var v4IP, v6IP, mac string
 	portName := fmt.Sprintf("node-%s", key)
 	if node.Annotations[util.AllocatedAnnotation] == "true" && node.Annotations[util.IpAddressAnnotation] != "" && node.Annotations[util.MacAddressAnnotation] != "" {
@@ -317,6 +322,10 @@ func (c *Controller) handleUpdateNode(key string) error {
 		return err
 	}
 
+	if err := c.checkChassisDupl(node); err != nil {
+		return err
+	}
+
 	for _, subnet := range subnets {
 		if util.GatewayContains(subnet.Spec.GatewayNode, node.Name) {
 			if err := c.reconcileGateway(subnet); err != nil {
@@ -485,3 +494,24 @@ func (c *Controller) checkNodeEcmpRouteExist(nodeIp, cidrBlock string) (bool, er
 	}
 	return false, nil
 }
+
+func (c *Controller) checkChassisDupl(node *v1.Node) error {
+	// notice that multiple chassises may arise and we are not prepared
+	chassisAdd, err := c.ovnClient.GetChassis(node.Name)
+	if err != nil {
+		klog.Errorf("failed to get node %s chassisID, %v", node.Name, err)
+		return err
+	}
+
+	chassisAnn := node.Annotations[util.ChassisAnnotation]
+	if chassisAnn != "" && chassisAnn != chassisAdd {
+		klog.Errorf("duplicate chassis for node %s and new chassis %s", node.Name, chassisAdd)
+		if err := c.ovnClient.DeleteChassis(node.Name); err != nil {
+			klog.Errorf("failed to delete chassis for node %s %v", node.Name, err)
+			return err
+		}
+	}
+
+	klog.V(3).Infof("finish check chassis, add %s and ann %s", chassisAdd, chassisAnn)
+	return nil
+}

pkg/util/const.go

@@ -15,6 +15,7 @@ const (
 	BgpAnnotation        = "ovn.kubernetes.io/bgp"
 	SnatAnnotation       = "ovn.kubernetes.io/snat"
 	EipAnnotation        = "ovn.kubernetes.io/eip"
+	ChassisAnnotation    = "ovn.kubernetes.io/chassis"
 
 	VpcNatGatewayAnnotation     = "ovn.kubernetes.io/vpc_nat_gw"
 	VpcNatGatewayInitAnnotation = "ovn.kubernetes.io/vpc_nat_gw_init"
@@ -97,4 +98,9 @@ const (
 
 	EcmpRouteType   = "ecmp"
 	NormalRouteType = "normal"
+
+	ChassisLoc     = "/etc/openvswitch/system-id.conf"
+	HostnameEnv    = "KUBE_NODE_NAME"
+	ChasRetryTime  = 5
+	ChasRetryIntev = 1
 )