From f139e99d052bb4cf12d1b18229cb96d3a5c74811 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Wed, 14 Feb 2024 08:47:25 -0800 Subject: [PATCH 1/2] fix: fixing CNS IP releae for azure CNI in case of managed endpoint state. --- cns/restserver/ipam.go | 9 +++++++-- cns/service/main.go | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index a328ebaa5e..37d3947c23 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -713,8 +713,13 @@ func (service *HTTPRestService) releaseIPConfigs(podInfo cns.PodInfo) error { service.Lock() defer service.Unlock() ipsToBeReleased := make([]cns.IPConfigurationStatus, 0) - - for i, ipID := range service.PodIPIDByPodInterfaceKey[podInfo.Key()] { + key := podInfo.Key() + if _, isMapContainsKey := service.PodIPIDByPodInterfaceKey[podInfo.Key()]; !isMapContainsKey { + // the special case for azure CNI with managed endpoint state since the podInfo.Key() is in containeID-eth0 format and we need to use full ContainerID as well + key = podInfo.InfraContainerID() + } + logger.Printf("[releaseIPConfigs] Released pod with key %s", key) + for i, ipID := range service.PodIPIDByPodInterfaceKey[key] { if ipID != "" { if ipconfig, isExist := service.PodIPConfigState[ipID]; isExist { ipsToBeReleased = append(ipsToBeReleased, ipconfig) diff --git a/cns/service/main.go b/cns/service/main.go index 3847c2c733..626aca21dd 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1244,6 +1244,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err = PopulateCNSEndpointState(httpRestServiceImplementation.EndpointStateStore); err != nil { return errors.Wrap(err, "failed to create CNS EndpointState From CNI") } + if err = httpRestServiceImplementation.EndpointStateStore.Read(restserver.EndpointStoreKey, &httpRestServiceImplementation.EndpointState); err != nil { + return errors.Wrap(err, "Failed to restore endpoint state") + } } var podInfoByIPProvider cns.PodInfoByIPProvider From 13c25608075cbd978fda043914c131171c7e39aa Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 15 Feb 2024 11:51:32 -0800 Subject: [PATCH 2/2] fix: changing PodInfo Key to InfraContainerID when Mange Endpoint State is enabled. --- cns/NetworkContainerContract.go | 12 ++++++++++-- cns/restserver/ipam.go | 9 ++------- cns/service/main.go | 5 +++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 9462ca3053..a0bd6756d7 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -180,6 +180,7 @@ type podInfoScheme int const ( KubernetesPodInfoScheme podInfoScheme = iota InterfaceIDPodInfoScheme + InfraIDPodInfoScheme ) // PodInfo represents the object that we are providing network for. @@ -249,11 +250,18 @@ func (p *podInfo) InterfaceID() string { // orchestrator pod name and namespace. if the Version is interfaceID, key is // composed of the CNI interfaceID, which is generated from the CRI infra // container ID and the pod net ns primary interface name. +// If the version in InfraContainerID then the key is containerID. func (p *podInfo) Key() string { - if p.Version == InterfaceIDPodInfoScheme { + switch p.Version { + case InfraIDPodInfoScheme: + return p.PodInfraContainerID + case InterfaceIDPodInfoScheme: return p.PodInterfaceID + case KubernetesPodInfoScheme: + return p.PodName + ":" + p.PodNamespace + default: + return p.PodName + ":" + p.PodNamespace } - return p.PodName + ":" + p.PodNamespace } func (p *podInfo) Name() string { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 37d3947c23..fc7be99059 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -713,13 +713,8 @@ func (service *HTTPRestService) releaseIPConfigs(podInfo cns.PodInfo) error { service.Lock() defer service.Unlock() ipsToBeReleased := make([]cns.IPConfigurationStatus, 0) - key := podInfo.Key() - if _, isMapContainsKey := service.PodIPIDByPodInterfaceKey[podInfo.Key()]; !isMapContainsKey { - // the special case for azure CNI with managed endpoint state since the podInfo.Key() is in containeID-eth0 format and we need to use full ContainerID as well - key = podInfo.InfraContainerID() - } - logger.Printf("[releaseIPConfigs] Released pod with key %s", key) - for i, ipID := range service.PodIPIDByPodInterfaceKey[key] { + logger.Printf("[releaseIPConfigs] Releasing pod with key %s", podInfo.Key()) + for i, ipID := range service.PodIPIDByPodInterfaceKey[podInfo.Key()] { if ipID != "" { if ipconfig, isExist := service.PodIPConfigState[ipID]; isExist { ipsToBeReleased = append(ipsToBeReleased, ipconfig) diff --git a/cns/service/main.go b/cns/service/main.go index 626aca21dd..43af60290d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -843,7 +843,7 @@ func main() { // in this case, cns maintains state with containerid as key and so in-memory cache can lookup // and update based on container id. if cnsconfig.ManageEndpointState { - cns.GlobalPodInfoScheme = cns.InterfaceIDPodInfoScheme + cns.GlobalPodInfoScheme = cns.InfraIDPodInfoScheme } logger.Printf("Set GlobalPodInfoScheme %v (InitializeFromCNI=%t)", cns.GlobalPodInfoScheme, cnsconfig.InitializeFromCNI) @@ -1244,8 +1244,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err = PopulateCNSEndpointState(httpRestServiceImplementation.EndpointStateStore); err != nil { return errors.Wrap(err, "failed to create CNS EndpointState From CNI") } + // endpoint state needs tobe loaded in memory so the subsequent Delete calls remove the state and release the IPs. if err = httpRestServiceImplementation.EndpointStateStore.Read(restserver.EndpointStoreKey, &httpRestServiceImplementation.EndpointState); err != nil { - return errors.Wrap(err, "Failed to restore endpoint state") + return errors.Wrap(err, "failed to restore endpoint state") } }