Skip to content

Commit

Permalink
fix ipam deletion (#3549)
Browse files Browse the repository at this point in the history 
* fix ipam deletion

---------

Signed-off-by: bobz965 <zhangbingbing2_yewu@cmss.chinamobile.com>
  • Loading branch information
@bobz965
bobz965 committed Dec 20, 2023
1 parent 40db901 commit e27aced
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 89 deletions.
161 changes: 78 additions & 83 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ func (c *Controller) enqueueAddPod(obj interface{}) {
klog.V(3).Infof("enqueue delete pod %s", key)
c.deletePodQueue.Add(obj)
}
if isVmPod && c.isVmPodToDel(p, vmName) {
if isVmPod && c.isVmToDel(p, vmName) {
klog.V(3).Infof("enqueue delete pod %s", key)
c.deletePodQueue.Add(obj)
}
Expand Down Expand Up @@ -315,7 +315,7 @@ func (c *Controller) enqueueUpdatePod(oldObj, newObj interface{}) {
}()
return
}
if isVmPod && c.isVmPodToDel(newPod, vmName) {
if isVmPod && c.isVmToDel(newPod, vmName) {
go func() {
klog.V(3).Infof("enqueue delete pod %s", key)
time.Sleep(time.Duration(*newPod.Spec.TerminationGracePeriodSeconds) * time.Second)
Expand Down Expand Up @@ -568,19 +568,11 @@ func (c *Controller) changeVMSubnet(vmName, namespace, providerName, subnetName
return err
}
}
ports, err := c.ovnClient.ListPodLogicalSwitchPorts(key)
if err != nil {
klog.Errorf("failed to list lsps of pod '%s', %v", pod.Name, err)
klog.Infof("gc logical switch port %s", key)
if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(key); err != nil {
klog.Errorf("failed to delete lsp %s, %v", key, err)
return err
}
for _, port := range ports {
// when lsp is deleted, the port of pod is deleted from any port-group automatically.
klog.Infof("gc logical switch port %s", port.Name)
if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(port.Name); err != nil {
klog.Errorf("failed to delete lsp %s, %v", port.Name, err)
return err
}
}
c.ipam.ReleaseAddressByPod(key, subnetName)
}
}
Expand Down Expand Up @@ -622,6 +614,8 @@ func (c *Controller) handleAddPod(key string) error {
pod.Annotations = map[string]string{}
}
isVmPod, vmName := isVmPod(pod)
podType := getPodType(pod)
// todo: isVmPod and getPodType has duplicated logic

// Avoid create lsp for already running pod in ovn-nb when controller restart
for _, podNet := range needAllocateSubnets(pod, podNets) {
Expand Down Expand Up @@ -655,14 +649,7 @@ func (c *Controller) handleAddPod(key string) error {
return err
}

podType := getPodType(pod)
podName := c.getNameByPod(pod)
if err := c.createOrUpdateCrdIPs(podName, ipStr, mac, subnet.Name, pod.Namespace, pod.Spec.NodeName, podNet.ProviderName, podType); err != nil {
err = fmt.Errorf("failed to create ips CR %s.%s: %v", podName, pod.Namespace, err)
klog.Error(err)
return err
}

if podNet.Type != providerTypeIPAM {
if (subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway) && subnet.Spec.Vpc != "" {
pod.Annotations[fmt.Sprintf(util.LogicalRouterAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.Vpc
Expand Down Expand Up @@ -719,6 +706,12 @@ func (c *Controller) handleAddPod(key string) error {
c.syncVirtualPortsQueue.Add(podNet.Subnet.Name)
}
}
// CreatePort may fail, so put ip cr creation after CreatePort
if err := c.createOrUpdateCrdIPs(podName, ipStr, mac, subnet.Name, pod.Namespace, pod.Spec.NodeName, podNet.ProviderName, podType); err != nil {
err = fmt.Errorf("failed to create ips CR %s.%s: %v", podName, pod.Namespace, err)
klog.Error(err)
return err
}
}

patch, err := util.GenerateMergePatchPayload(oriPod, pod)
Expand Down Expand Up @@ -758,46 +751,7 @@ func (c *Controller) handleDeletePod(pod *v1.Pod) error {
// Pod with same name exists, just return here
return nil
}

podKey := fmt.Sprintf("%s/%s", pod.Namespace, podName)
ports, err := c.ovnClient.ListPodLogicalSwitchPorts(podKey)
if err != nil {
klog.Errorf("failed to list lsps of pod '%s', %v", pod.Name, err)
return err
}

if len(ports) != 0 {
addresses := c.ipam.GetPodAddress(podKey)
for _, address := range addresses {
if strings.TrimSpace(address.Ip) == "" {
continue
}
subnet, err := c.subnetsLister.Get(address.Subnet.Name)
if k8serrors.IsNotFound(err) {
continue
} else if err != nil {
return err
}
vpc, err := c.vpcsLister.Get(subnet.Spec.Vpc)
if k8serrors.IsNotFound(err) {
continue
} else if err != nil {
return err
}
// If pod has snat or eip, also need delete staticRoute when delete pod
if vpc.Name == util.DefaultVpc {
if err := c.ovnLegacyClient.DeleteStaticRoute(address.Ip, vpc.Name); err != nil {
return err
}
}
if exGwEnabled == "true" {
if err := c.ovnLegacyClient.DeleteNatRule(address.Ip, vpc.Name); err != nil {
return err
}
}
}
}

var keepIpCR bool
if ok, sts := isStatefulSetPod(pod); ok {
toDel := isStatefulSetPodToDel(c.config.KubeClient, pod, sts)
Expand All @@ -812,44 +766,84 @@ func (c *Controller) handleDeletePod(pod *v1.Pod) error {
}
isVmPod, vmName := isVmPod(pod)
if isVmPod && c.config.EnableKeepVmIP {
toDel := c.isVmPodToDel(pod, vmName)
vmToBeDel := c.isVmToDel(pod, vmName)
delete, err := appendCheckPodToDel(c, pod, vmName, util.VmInstance)
if pod.DeletionTimestamp != nil {
// triggered by delete event
if !(toDel || (delete && err == nil)) {
if !(vmToBeDel || (delete && err == nil)) {
return nil
}
klog.Infof("delete vm pod %s", podName)
}
}

for _, port := range ports {
sgs, err := c.getPortSg(&port)
if err != nil {
klog.Warningf("failed to get port '%s' sg, %v", port.Name, err)
}
// when lsp is deleted, the port of pod is deleted from any port-group automatically.
klog.Infof("gc logical switch port %s", port.Name)
if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(port.Name); err != nil {
klog.Errorf("failed to delete lsp %s, %v", port.Name, err)
return err
}
for _, sg := range sgs {
c.syncSgPortsQueue.Add(sg)
}
}
klog.Infof("release all ip address for deleting pod %s", key)
c.ipam.ReleaseAddressByPod(podKey, "")
podNets, err := c.getPodKubeovnNets(pod)
if err != nil {
klog.Errorf("failed to get pod nets %v", err)
}
if !keepIpCR {
// delete lsp, ip cr, ipam, sg
ports, err := c.ovnClient.ListPodLogicalSwitchPorts(podKey)
if err != nil {
klog.Errorf("failed to list lsps of pod '%s', %v", pod.Name, err)
return err
}
if len(ports) != 0 {
addresses := c.ipam.GetPodAddress(podKey)
for _, address := range addresses {
if strings.TrimSpace(address.Ip) == "" {
continue
}
subnet, err := c.subnetsLister.Get(address.Subnet.Name)
if k8serrors.IsNotFound(err) {
continue
} else if err != nil {
klog.Errorf("failed to get subnet %s, %v", address.Subnet.Name, err)
return err
}
vpc, err := c.vpcsLister.Get(subnet.Spec.Vpc)
if k8serrors.IsNotFound(err) {
continue
} else if err != nil {
klog.Errorf("failed to get vpc %s, %v", subnet.Spec.Vpc, err)
return err
}
// If pod has snat or eip, also need delete staticRoute when delete pod
if vpc.Name == util.DefaultVpc {
if err := c.ovnLegacyClient.DeleteStaticRoute(address.Ip, vpc.Name); err != nil {
klog.Errorf("failed to delete static route %s, %v", address.Ip, err)
return err
}
}
if exGwEnabled == "true" {
if err := c.ovnLegacyClient.DeleteNatRule(address.Ip, vpc.Name); err != nil {
klog.Errorf("failed to delete nat rule %s, %v", address.Ip, err)
return err
}
}
}
}
for _, port := range ports {
sgs, err := c.getPortSg(&port)
if err != nil {
klog.Warningf("failed to get port '%s' sg, %v", port.Name, err)
}
// when lsp is deleted, the port of pod is deleted from any port-group automatically.
klog.Infof("gc logical switch port %s", port.Name)
if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(port.Name); err != nil {
klog.Errorf("failed to delete lsp %s, %v", port.Name, err)
return err
}
for _, sg := range sgs {
c.syncSgPortsQueue.Add(sg)
}
}
klog.Infof("release all ip address for deleting pod %s", key)
for _, podNet := range podNets {
if err = c.deleteCrdIPs(podName, pod.Namespace, podNet.ProviderName); err != nil {
klog.Errorf("failed to delete ip for pod %s, %v, please delete manually", pod.Name, err)
}
}
c.ipam.ReleaseAddressByPod(podKey, "")
if pod.Annotations[util.VipAnnotation] != "" {
if err = c.releaseVip(pod.Annotations[util.VipAnnotation]); err != nil {
klog.Errorf("failed to clean label from vip %s, %v", pod.Annotations[util.VipAnnotation], err)
Expand Down Expand Up @@ -1114,7 +1108,7 @@ func isStatefulSetPodToDel(c kubernetes.Interface, pod *v1.Pod, statefulSetName
klog.Errorf("failed to parse %s to int", numStr)
return false
}

// down scaled
return index >= int64(*ss.Spec.Replicas)
}

Expand Down Expand Up @@ -1567,8 +1561,9 @@ func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind
}

if !ownerRefSubnetExist {
subnetNames := podNs.Annotations[util.LogicalSwitchAnnotation]
if subnetNames != "" && pod.Annotations[util.LogicalSwitchAnnotation] != "" && !util.ContainsString(strings.Split(subnetNames, ","), strings.TrimSpace(pod.Annotations[util.LogicalSwitchAnnotation])) {
nsSubnetNames := podNs.Annotations[util.LogicalSwitchAnnotation]
// check if pod use the subnet of its ns
if nsSubnetNames != "" && pod.Annotations[util.LogicalSwitchAnnotation] != "" && !util.ContainsString(strings.Split(nsSubnetNames, ","), strings.TrimSpace(pod.Annotations[util.LogicalSwitchAnnotation])) {
klog.Infof("ns %s annotation subnet is %s, which is inconstant with subnet for pod %s, delete pod", pod.Namespace, podNs.Annotations[util.LogicalSwitchAnnotation], pod.Name)
return true, nil
}
Expand Down Expand Up @@ -1659,7 +1654,7 @@ func isOwnsByTheVM(vmi metav1.Object) (bool, string) {
return false, ""
}

func (c *Controller) isVmPodToDel(pod *v1.Pod, vmiName string) bool {
func (c *Controller) isVmToDel(pod *v1.Pod, vmiName string) bool {
var (
vmiAlive bool
vmName string
Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/pod_iptables_eip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (c *Controller) enqueueAddPodAnnotatedIptablesEip(obj interface{}) {
c.delPodAnnotatedIptablesEipQueue.Add(obj)
return
}
if isVmPod && c.isVmPodToDel(p, vmName) {
if isVmPod && c.isVmToDel(p, vmName) {
klog.V(3).Infof("enqueue delete pod annotated iptables eip %s", eipName)
c.delPodAnnotatedIptablesEipQueue.Add(obj)
return
Expand Down Expand Up @@ -106,7 +106,7 @@ func (c *Controller) enqueueUpdatePodAnnotatedIptablesEip(oldObj, newObj interfa
c.delPodAnnotatedIptablesEipQueue.Add(newObj)
return
}
if c.config.EnableKeepVmIP && isVmPod && c.isVmPodToDel(newPod, vmName) {
if c.config.EnableKeepVmIP && isVmPod && c.isVmToDel(newPod, vmName) {
c.delPodAnnotatedIptablesEipQueue.Add(newObj)
return
}
Expand Down Expand Up @@ -137,7 +137,7 @@ func (c *Controller) enqueueDeletePodAnnotatedIptablesEip(obj interface{}) {
return
}
} else if c.config.EnableKeepVmIP && isVmPod {
if c.isVmPodToDel(p, vmName) {
if c.isVmToDel(p, vmName) {
c.delPodAnnotatedIptablesEipQueue.Add(obj)
return
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/pod_iptables_fip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func (c *Controller) enqueueAddPodAnnotatedIptablesFip(obj interface{}) {
c.delPodAnnotatedIptablesFipQueue.Add(obj)
return
}
if isVmPod && c.isVmPodToDel(p, vmName) {
if isVmPod && c.isVmToDel(p, vmName) {
c.delPodAnnotatedIptablesFipQueue.Add(obj)
return
}
Expand Down Expand Up @@ -94,7 +94,7 @@ func (c *Controller) enqueueUpdatePodAnnotatedIptablesFip(oldObj, newObj interfa
c.delPodAnnotatedIptablesFipQueue.Add(newObj)
return
}
if c.config.EnableKeepVmIP && isVmPod && c.isVmPodToDel(newPod, vmName) {
if c.config.EnableKeepVmIP && isVmPod && c.isVmToDel(newPod, vmName) {
c.delPodAnnotatedIptablesFipQueue.Add(newObj)
return
}
Expand Down Expand Up @@ -125,7 +125,7 @@ func (c *Controller) enqueueDeletePodAnnotatedIptablesFip(obj interface{}) {
return
}
} else if c.config.EnableKeepVmIP && isVmPod {
if c.isVmPodToDel(p, vmName) {
if c.isVmToDel(p, vmName) {
c.delPodAnnotatedIptablesFipQueue.Add(obj)
return
}
Expand Down

0 comments on commit e27aced

Please sign in to comment.