Inform opflex-agent that opflexOdev is deleted

pkg/controller/controller.go

@@ -151,6 +151,7 @@ type AciController struct {
 
 	nodeServiceMetaCache map[string]*nodeServiceMeta
 	nodeACIPod           map[string]aciPodAnnot
+	nodeACIPodAnnot      map[string]aciPodAnnot
 	nodeOpflexDevice     map[string]apicapi.ApicSlice
 	nodePodNetCache      map[string]*nodePodNetMeta
 	serviceMetaCache     map[string]*serviceMeta
@@ -393,6 +394,7 @@ func NewController(config *ControllerConfig, env Environment, log *logrus.Logger
 		nodeServiceIps:          newNetIps(),
 
 		nodeACIPod:       make(map[string]aciPodAnnot),
+		nodeACIPodAnnot:  make(map[string]aciPodAnnot),
 		nodeOpflexDevice: make(map[string]apicapi.ApicSlice),
 
 		nodeServiceMetaCache:    make(map[string]*nodeServiceMeta),
@@ -847,7 +849,10 @@ func (cont *AciController) syncNodeAciPods(stopCh <-chan struct{}, seconds time.
 	for {
 		select {
 		case <-ticker.C:
-			cont.checkChangeOfOdevAciPod()
+			cont.checkChangeOfOpflexOdevAciPod()
+			if cont.config.AciMultipod {
+				cont.checkChangeOfOdevAciPod()
+			}
 		case <-stopCh:
 			return
 		}

pkg/controller/environment.go

@@ -344,9 +344,7 @@ func (env *K8sEnvironment) PrepareRun(stopCh <-chan struct{}) error {
 		}
 		go cont.syncOpflexDevices(stopCh, 60)
 		go cont.syncDelayedEpSlices(stopCh, 1)
-		if cont.config.AciMultipod {
-			go cont.syncNodeAciPods(stopCh, 1)
-		}
+		go cont.syncNodeAciPods(stopCh, 1)
 	}
 	return nil
 }

pkg/controller/nodes.go

@@ -397,6 +397,21 @@ func (cont *AciController) nodeChanged(obj interface{}) {
 		}
 	}
 
+	nodeAciPodAnnotation, ok := cont.nodeACIPodAnnot[node.ObjectMeta.Name]
+	if ok {
+		nodeAciPod := nodeAciPodAnnotation.aciPod
+		aciPodAnn := node.ObjectMeta.Annotations[metadata.NodeAciPodAnnotation]
+		if cont.nodeSyncEnabled {
+			if aciPodAnn != nodeAciPod && nodeAciPod != "" {
+				node.ObjectMeta.Annotations[metadata.NodeAciPodAnnotation] = nodeAciPod
+				nodeUpdated = true
+			}
+		}
+	} else {
+		var annot aciPodAnnot
+		cont.nodeACIPodAnnot[node.ObjectMeta.Name] = annot
+	}
+
 	if cont.config.AciMultipod {
 		nodeAciPodAnnot, ok := cont.nodeACIPod[node.ObjectMeta.Name]
 		if ok {
@@ -508,6 +523,7 @@ func (cont *AciController) nodeDeleted(obj interface{}) {
 		cont.log.Debug("Node deleted from nodePodNetCache: ", node.ObjectMeta.Name)
 	}
 
+	delete(cont.nodeACIPodAnnot, node.ObjectMeta.Name)
 	np, ok := obj.(*nodePodIf.NodePodIF)
 	if !ok {
 		deletedState, ok := obj.(cache.DeletedFinalStateUnknown)

pkg/controller/services.go

@@ -425,6 +425,82 @@ func (cont *AciController) createAciPodAnnotation(node string) (aciPodAnnot, err
 	return nodeAciPodAnnot, fmt.Errorf("Failed to get annotation")
 }
 
+func (cont *AciController) createNodeAciPodAnnotation(node string) (aciPodAnnot, error) {
+	odevCount, fabricPathDn := cont.getOpflexOdevCount(node)
+	nodeAciPodAnnot := cont.nodeACIPodAnnot[node]
+	isSingleOdev := nodeAciPodAnnot.isSingleOpflexOdev
+	if (odevCount == 0) ||
+		(odevCount == 1 && !isSingleOdev) {
+		if nodeAciPodAnnot.aciPod != "none" {
+			nodeAciPodAnnot.aciPod = "none"
+		}
+		if odevCount == 1 {
+			nodeAciPodAnnot.isSingleOpflexOdev = true
+		}
+		nodeAciPodAnnot.connectTime = time.Time{}
+		return nodeAciPodAnnot, nil
+	} else if (odevCount == 2) ||
+		(odevCount == 1 && isSingleOdev) {
+		nodeAciPod := nodeAciPodAnnot.aciPod
+		if odevCount == 2 {
+			nodeAciPodAnnot.isSingleOpflexOdev = false
+		}
+		if odevCount == 1 && nodeAciPod == "none" {
+			if nodeAciPodAnnot.connectTime.IsZero() {
+				nodeAciPodAnnot.connectTime = time.Now()
+				return nodeAciPodAnnot, nil
+			} else {
+				currentTime := time.Now()
+				diff := currentTime.Sub(nodeAciPodAnnot.connectTime)
+				if diff.Seconds() < float64(10) {
+					return nodeAciPodAnnot, nil
+				}
+			}
+		}
+		nodeAciPodAnnot.connectTime = time.Time{}
+		pathSlice := strings.Split(fabricPathDn, "/")
+		if len(pathSlice) > 1 {
+
+			nodeAciPodAnnot.aciPod = pathSlice[1]
+			return nodeAciPodAnnot, nil
+		} else {
+			cont.log.Error("Invalid fabricPathDn of opflexOdev of node ", node)
+			return nodeAciPodAnnot, fmt.Errorf("Invalid fabricPathDn of opflexOdev")
+		}
+	}
+	return nodeAciPodAnnot, fmt.Errorf("Failed to get annotation")
+}
+
+func (cont *AciController) checkChangeOfOpflexOdevAciPod() {
+	var nodeAnnotationUpdates []string
+	cont.apicConn.SyncMutex.Lock()
+	syncDone := cont.apicConn.SyncDone
+	cont.apicConn.SyncMutex.Unlock()
+
+	if !syncDone {
+		return
+	}
+
+	cont.indexMutex.Lock()
+	for node := range cont.nodeACIPodAnnot {
+		annot, err := cont.createNodeAciPodAnnotation(node)
+		if err != nil {
+			cont.log.Error(err.Error())
+		} else {
+			if annot != cont.nodeACIPodAnnot[node] {
+				cont.nodeACIPodAnnot[node] = annot
+				nodeAnnotationUpdates = append(nodeAnnotationUpdates, node)
+			}
+		}
+	}
+	cont.indexMutex.Unlock()
+	if len(nodeAnnotationUpdates) > 0 {
+		for _, updatednode := range nodeAnnotationUpdates {
+			go cont.env.NodeAnnotationChanged(updatednode)
+		}
+	}
+}
+
 func (cont *AciController) checkChangeOfOdevAciPod() {
 	var nodeAnnotationUpdates []string
 	cont.apicConn.SyncMutex.Lock()

pkg/hostagent/agent.go

@@ -86,6 +86,7 @@ type HostAgent struct {
 	rcPods                 *index.PodSelectorIndex
 	podNetAnnotation       string
 	aciPodAnnotation       string
+	nodeAciPodAnnotation   string
 	podIps                 *ipam.IpCache
 	usedIPs                map[string]string
 	netAttDefInformer      cache.SharedIndexInformer

pkg/hostagent/nodes.go

@@ -110,6 +110,14 @@ func (agent *HostAgent) nodeChanged(obj ...interface{}) {
 
 	agent.indexMutex.Lock()
 
+	nodeAciPod, acipodok := node.ObjectMeta.Annotations[metadata.NodeAciPodAnnotation]
+	if acipodok {
+		if agent.nodeAciPodAnnotation != nodeAciPod && nodeAciPod == "none" {
+			agent.informOpflexAgent(nodeAciPod)
+		}
+		agent.nodeAciPodAnnotation = nodeAciPod
+	}
+
 	if agent.config.AciMultipod {
 		aciPod, acipodok := node.ObjectMeta.Annotations[metadata.AciPodAnnotation]
 		if acipodok {

pkg/hostagent/opflex.go

@@ -115,6 +115,29 @@ func (agent *HostAgent) isIpSameSubnet(iface, subnet string) bool {
 	return false
 }
 
+func appendDateToFile(filename string) error {
+	f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	t := time.Now()
+	if _, err = f.WriteString(t.String() + "\n"); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (agent *HostAgent) informOpflexAgent(acipod string) {
+	resetFile := "/usr/local/var/lib/opflex-agent-ovs/reboot-conf.d/reset.conf"
+	err := appendDateToFile(resetFile)
+	if err != nil {
+		agent.log.Error("Failed to inform opflex-agent about opflexOdev disconnect ", err)
+		return
+	}
+	agent.log.Debug("Informed opflex-agent about opflexOdev disconnect")
+}
+
 func (agent *HostAgent) doDhcpRenew(aciPodSubnet string) {
 	retryCount := agent.config.DhcpRenewMaxRetryCount
 	dhcpDelay := time.Duration(agent.config.DhcpDelay)

pkg/metadata/annotations.go

@@ -60,6 +60,7 @@ const ServiceContractScopeAnnotation = "opflex.cisco.com/ext_service_contract_sc
 const PodNetworkRangeAnnotation = "opflex.cisco.com/pod-network-ranges"
 
 const AciPodAnnotation = "opflex.cisco.com/aci-pod"
+const NodeAciPodAnnotation = "opflex.cisco.com/node-aci-pod"
 
 // Annotation for endpoint group designation for pod, deployment, etc.
 const EgAnnotation = "opflex.cisco.com/endpoint-group"