From af16c760b726fa36d2ffabdf0a3a882644067ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=A5=96=E5=BB=BA?= Date: Thu, 27 Apr 2023 18:00:47 +0800 Subject: [PATCH] fix ovn lb gc (#2728) --- pkg/controller/endpoint.go | 7 ++--- pkg/controller/gc.go | 50 +++++++++++++++++++------------- pkg/controller/network_policy.go | 5 +--- pkg/util/k8s.go | 8 +++++ 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/pkg/controller/endpoint.go b/pkg/controller/endpoint.go index 1b12631873a..1649f2b48ee 100644 --- a/pkg/controller/endpoint.go +++ b/pkg/controller/endpoint.go @@ -115,11 +115,8 @@ func (c *Controller) handleUpdateEndpoint(key string) error { } svc := orisvc.DeepCopy() - clusterIPs := svc.Spec.ClusterIPs - if len(clusterIPs) == 0 && svc.Spec.ClusterIP != "" && svc.Spec.ClusterIP != v1.ClusterIPNone { - clusterIPs = []string{svc.Spec.ClusterIP} - } - if len(clusterIPs) == 0 || clusterIPs[0] == v1.ClusterIPNone { + clusterIPs := util.ServiceClusterIPs(*svc) + if len(clusterIPs) == 0 { return nil } diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index e30101bed7c..af6c8f4183a 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -322,6 +323,7 @@ func (c *Controller) markAndCleanLSP() error { func (c *Controller) gcLoadBalancer() error { klog.Infof("start to gc loadbalancers") + start := time.Now() if !c.config.EnableLb { // remove lb from logical switch vpcs, err := c.vpcsLister.List(labels.Everything()) @@ -385,24 +387,27 @@ func (c *Controller) gcLoadBalancer() error { klog.Errorf("failed to list svc, %v", err) return err } - tcpVips := []string{} - udpVips := []string{} - tcpSessionVips := []string{} - udpSessionVips := []string{} + tcpVips := make(map[string]struct{}, len(svcs)*2) + udpVips := make(map[string]struct{}, len(svcs)*2) + tcpSessionVips := make(map[string]struct{}, len(svcs)*2) + udpSessionVips := make(map[string]struct{}, len(svcs)*2) for _, svc := range svcs { - ip := svc.Spec.ClusterIP - for _, port := range svc.Spec.Ports { - if port.Protocol == corev1.ProtocolTCP { - if svc.Spec.SessionAffinity == corev1.ServiceAffinityClientIP { - tcpSessionVips = append(tcpSessionVips, fmt.Sprintf("%s:%d", ip, port.Port)) - } else { - tcpVips = append(tcpVips, fmt.Sprintf("%s:%d", ip, port.Port)) - } - } else { - if svc.Spec.SessionAffinity == corev1.ServiceAffinityClientIP { - udpSessionVips = append(udpSessionVips, fmt.Sprintf("%s:%d", ip, port.Port)) + ips := util.ServiceClusterIPs(*svc) + for _, ip := range ips { + for _, port := range svc.Spec.Ports { + vip := util.JoinHostPort(ip, port.Port) + if port.Protocol == corev1.ProtocolTCP { + if svc.Spec.SessionAffinity == corev1.ServiceAffinityClientIP { + tcpSessionVips[vip] = struct{}{} + } else { + tcpVips[vip] = struct{}{} + } } else { - udpVips = append(udpVips, fmt.Sprintf("%s:%d", ip, port.Port)) + if svc.Spec.SessionAffinity == corev1.ServiceAffinityClientIP { + udpSessionVips[vip] = struct{}{} + } else { + udpVips[vip] = struct{}{} + } } } } @@ -430,7 +435,8 @@ func (c *Controller) gcLoadBalancer() error { return err } for vip := range vips { - if !util.IsStringIn(vip, tcpVips) { + if _, ok := tcpVips[vip]; !ok { + klog.Infof("gc vip %s in LB %s", vip, tcpLb) err := c.ovnLegacyClient.DeleteLoadBalancerVip(vip, tcpLb) if err != nil { klog.Errorf("failed to delete vip %s from tcp lb %s, %v", vip, tcpLb, err) @@ -451,7 +457,8 @@ func (c *Controller) gcLoadBalancer() error { return err } for vip := range vips { - if !util.IsStringIn(vip, tcpSessionVips) { + if _, ok := tcpSessionVips[vip]; !ok { + klog.Infof("gc vip %s in LB %s", vip, tcpSessLb) err := c.ovnLegacyClient.DeleteLoadBalancerVip(vip, tcpSessLb) if err != nil { klog.Errorf("failed to delete vip %s from tcp session lb %s, %v", vip, tcpSessLb, err) @@ -473,7 +480,8 @@ func (c *Controller) gcLoadBalancer() error { return err } for vip := range vips { - if !util.IsStringIn(vip, udpVips) { + if _, ok := udpVips[vip]; !ok { + klog.Infof("gc vip %s in LB %s", vip, udpLb) err := c.ovnLegacyClient.DeleteLoadBalancerVip(vip, udpLb) if err != nil { klog.Errorf("failed to delete vip %s from tcp lb %s, %v", vip, udpLb, err) @@ -495,7 +503,8 @@ func (c *Controller) gcLoadBalancer() error { return err } for vip := range vips { - if !util.IsStringIn(vip, udpSessionVips) { + if _, ok := udpSessionVips[vip]; !ok { + klog.Infof("gc vip %s in LB %s", vip, udpSessLb) err := c.ovnLegacyClient.DeleteLoadBalancerVip(vip, udpSessLb) if err != nil { klog.Errorf("failed to delete vip %s from udp session lb %s, %v", vip, udpSessLb, err) @@ -523,6 +532,7 @@ func (c *Controller) gcLoadBalancer() error { return err } } + klog.Infof("took %.2fs to gc load balancers", time.Since(start).Seconds()) return nil } diff --git a/pkg/controller/network_policy.go b/pkg/controller/network_policy.go index 316b3433956..997bba0bc96 100644 --- a/pkg/controller/network_policy.go +++ b/pkg/controller/network_policy.go @@ -743,10 +743,7 @@ func svcMatchPods(svcs []*corev1.Service, pod *corev1.Pod, protocol string) ([]s return nil, err } if isMatch { - clusterIPs := svc.Spec.ClusterIPs - if len(clusterIPs) == 0 && svc.Spec.ClusterIP != "" && svc.Spec.ClusterIP != corev1.ClusterIPNone { - clusterIPs = []string{svc.Spec.ClusterIP} - } + clusterIPs := util.ServiceClusterIPs(*svc) protocolClusterIPs := getProtocolSvcIp(clusterIPs, protocol) if len(protocolClusterIPs) != 0 { matchSvcs = append(matchSvcs, protocolClusterIPs...) diff --git a/pkg/util/k8s.go b/pkg/util/k8s.go index 885a4a5657e..176980216ea 100644 --- a/pkg/util/k8s.go +++ b/pkg/util/k8s.go @@ -44,3 +44,11 @@ func GetNodeInternalIP(node v1.Node) (ipv4, ipv6 string) { return SplitStringIP(strings.Join(ips, ",")) } + +func ServiceClusterIPs(svc v1.Service) []string { + ips := svc.Spec.ClusterIPs + if len(ips) == 0 && svc.Spec.ClusterIP != v1.ClusterIPNone && svc.Spec.ClusterIP != "" { + ips = []string{svc.Spec.ClusterIP} + } + return ips +}