From 39217f019160f09de403a278b8b119f4168f1954 Mon Sep 17 00:00:00 2001 From: backguynn Date: Thu, 27 Jun 2024 16:14:58 +0900 Subject: [PATCH 1/5] add debug logs --- .../manager/loadbalancer/loadbalancer.go | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go index 7955c77..28c38d7 100644 --- a/pkg/agent/manager/loadbalancer/loadbalancer.go +++ b/pkg/agent/manager/loadbalancer/loadbalancer.go @@ -1373,21 +1373,25 @@ func (m *Manager) getIngressSvcPairs(service *corev1.Service, addrType string, l // k8s service has ingress IP already if len(inSPairs) >= 1 { + klog.V(4).Infof("getIngressSvcPairs: service %s has servicePairs: %v", cacheKey, inSPairs) + klog.V(4).Infof("getIngressSvcPairs: service %s has externalIP: %v", cacheKey, service.Status.LoadBalancer.Ingress) checkSvcPortLoop: for _, inSPair := range inSPairs { hasExtIPAllocated = true for _, sp := range lbCacheEntry.LbServicePairs { if GenSPKey(inSPair.IPString, uint16(inSPair.Port), inSPair.Protocol) == GenSPKey(sp.ExternalIP, sp.Port, sp.Protocol) { - sp := SvcPair{sp.ExternalIP, int32(sp.Port), sp.Protocol, sp.InRange, sp.StaticIP, sp.IdentIPAM, false, inSPair.K8sSvcPort} - sPairs = append(sPairs, sp) + oldsp := SvcPair{sp.ExternalIP, int32(sp.Port), sp.Protocol, sp.InRange, sp.StaticIP, sp.IdentIPAM, false, inSPair.K8sSvcPort} + sPairs = append(sPairs, oldsp) + klog.V(4).Infof("getIngressSvcPairs: LB cache %s already has servicePairs: %v", cacheKey, sp) continue checkSvcPortLoop } } inRange, _, identStr := ipPool.CheckAndReserveIP(inSPair.IPString, cacheKey, uint32(inSPair.Port), inSPair.Protocol) - sp := SvcPair{inSPair.IPString, inSPair.Port, inSPair.Protocol, inRange, true, identStr, true, inSPair.K8sSvcPort} - sPairs = append(sPairs, sp) + newsp := SvcPair{inSPair.IPString, inSPair.Port, inSPair.Protocol, inRange, true, identStr, true, inSPair.K8sSvcPort} + klog.V(4).Infof("getIngressSvcPairs: LB cache %s is added servicePairs: %v", cacheKey, newsp) + sPairs = append(sPairs, newsp) } } @@ -1396,7 +1400,7 @@ func (m *Manager) getIngressSvcPairs(service *corev1.Service, addrType string, l // If hasExtIPAllocated is false, that means k8s service has no ingress IP if !hasExtIPAllocated { - var sp SvcPair + klog.V(4).Infof("getIngressSvcPairs: service %s has no externalIP: %v", cacheKey, service.Status.LoadBalancer.Ingress) checkServicePortLoop: for _, port := range service.Spec.Ports { proto := strings.ToLower(string(port.Protocol)) @@ -1404,23 +1408,27 @@ func (m *Manager) getIngressSvcPairs(service *corev1.Service, addrType string, l for _, sp := range lbCacheEntry.LbServicePairs { if sp.Port == uint16(portNum) && proto == sp.Protocol { - sp := SvcPair{sp.ExternalIP, int32(sp.Port), sp.Protocol, sp.InRange, sp.StaticIP, sp.IdentIPAM, false, port} - sPairs = append(sPairs, sp) + oldsp := SvcPair{sp.ExternalIP, int32(sp.Port), sp.Protocol, sp.InRange, sp.StaticIP, sp.IdentIPAM, false, port} + sPairs = append(sPairs, oldsp) + klog.V(4).Infof("getIngressSvcPairs: LB cache %s already has servicePairs: %v", cacheKey, sp) continue checkServicePortLoop } } newIP, identIPAM = ipPool.GetNewIPAddr(cacheKey, uint32(portNum), proto) if newIP == nil { - klog.Errorf("failed to generate external IP. IP Pool is full") + errMsg := fmt.Sprintf("failed to generate external IP. %s:%d:%s already used for %s", cacheKey, portNum, proto, identIPAM) + klog.Errorf(errMsg) klog.Exit("kube-loxilb cant run optimally anymore") - return nil, errors.New("failed to generate external IP. IP Pool is full"), hasExtIPAllocated + return nil, errors.New(errMsg), hasExtIPAllocated } - sp = SvcPair{newIP.String(), portNum, proto, true, false, identIPAM, true, port} - sPairs = append(sPairs, sp) + + klog.V(4).Infof("getIngressSvcPairs: service %s is generated new externalIP: %s", cacheKey, newIP.String()) + + newsp := SvcPair{newIP.String(), portNum, proto, true, false, identIPAM, true, port} + sPairs = append(sPairs, newsp) } } - //klog.Infof("Spairs: %v", sPairs) return sPairs, nil, hasExtIPAllocated } From c87316c3370b049bacacde1e5afb0ffb8ae37f30 Mon Sep 17 00:00:00 2001 From: backguynn Date: Thu, 27 Jun 2024 16:26:03 +0900 Subject: [PATCH 2/5] add debug logs --- pkg/agent/manager/loadbalancer/loadbalancer.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go index 28c38d7..82857eb 100644 --- a/pkg/agent/manager/loadbalancer/loadbalancer.go +++ b/pkg/agent/manager/loadbalancer/loadbalancer.go @@ -1501,8 +1501,8 @@ func (m *Manager) getIngressSecSvcPairs(service *corev1.Service, numSecondary in for _, sp := range lbCacheEntry.LbServicePairs { if sp.Port == uint16(portNum) && proto == sp.Protocol { - sp := SvcPair{sp.ExternalIP, int32(sp.Port), sp.Protocol, sp.InRange, sp.StaticIP, sp.IdentIPAM, false, port} - sPairs = append(sPairs, sp) + oldsp := SvcPair{sp.ExternalIP, int32(sp.Port), sp.Protocol, sp.InRange, sp.StaticIP, sp.IdentIPAM, false, port} + sPairs = append(sPairs, oldsp) continue checkServicePortLoop } } @@ -1513,12 +1513,13 @@ func (m *Manager) getIngressSecSvcPairs(service *corev1.Service, numSecondary in rpool := sipPools[j] rpool.ReturnIPAddr(sPairs[j].IPString, sPairs[j].IdentIPAM) } - klog.Errorf("failed to generate external secondary IP. IP Pool is full") + errMsg := fmt.Sprintf("failed to generate secondary external IP. %s:%d:%s already used for %s", cacheKey, portNum, proto, identIPAM) + klog.Errorf(errMsg) klog.Exit("kube-loxilb cant run optimally anymore") - return nil, errors.New("failed to generate external secondary IP. IP Pool is full") + return nil, errors.New(errMsg) } - sp := SvcPair{newIP.String(), portNum, proto, true, false, identIPAM, true, port} - sPairs = append(sPairs, sp) + newsp := SvcPair{newIP.String(), portNum, proto, true, false, identIPAM, true, port} + sPairs = append(sPairs, newsp) } } From 0f7647e743b4648b68d04a0d92630a272abfa2fc Mon Sep 17 00:00:00 2001 From: backguynn Date: Tue, 2 Jul 2024 15:26:41 +0900 Subject: [PATCH 3/5] add TLS annotation --- pkg/agent/manager/gatewayapi/const.go | 2 +- pkg/agent/manager/gatewayapi/gateway.go | 2 +- pkg/agent/manager/gatewayapi/tcproute.go | 2 +- pkg/agent/manager/gatewayapi/udproute.go | 2 +- pkg/agent/manager/loadbalancer/loadbalancer.go | 14 ++++++++++++++ pkg/api/lb.go | 1 + 6 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pkg/agent/manager/gatewayapi/const.go b/pkg/agent/manager/gatewayapi/const.go index 0af18f2..63b9d6f 100644 --- a/pkg/agent/manager/gatewayapi/const.go +++ b/pkg/agent/manager/gatewayapi/const.go @@ -23,7 +23,7 @@ const ( minRetryDelay = 2 * time.Second maxRetryDelay = 120 * time.Second defaultWorkers = 4 - contextTimeout = 5 * time.Second + contextTimeout = 30 * time.Second implementation = "kube-loxilb" finalizer = "loxilb.io" ) diff --git a/pkg/agent/manager/gatewayapi/gateway.go b/pkg/agent/manager/gatewayapi/gateway.go index ad1e02d..0a75e9b 100644 --- a/pkg/agent/manager/gatewayapi/gateway.go +++ b/pkg/agent/manager/gatewayapi/gateway.go @@ -78,7 +78,7 @@ func NewGatewayManager( gatewayLister: gatewayInformer.Lister(), gatewayListerSynced: gatewayInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "gatewayClass"), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "gateway"), } manager.gatewayInformer.Informer().AddEventHandler( diff --git a/pkg/agent/manager/gatewayapi/tcproute.go b/pkg/agent/manager/gatewayapi/tcproute.go index e61381b..ab18767 100644 --- a/pkg/agent/manager/gatewayapi/tcproute.go +++ b/pkg/agent/manager/gatewayapi/tcproute.go @@ -82,7 +82,7 @@ func NewTCPRouteManager( tcpRouteLister: tcpRouteInformer.Lister(), tcpRouteListerSynced: tcpRouteInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "gatewayClass"), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "tcpRoute"), } manager.tcpRouteInformer.Informer().AddEventHandler( diff --git a/pkg/agent/manager/gatewayapi/udproute.go b/pkg/agent/manager/gatewayapi/udproute.go index fa74aec..0e3feab 100644 --- a/pkg/agent/manager/gatewayapi/udproute.go +++ b/pkg/agent/manager/gatewayapi/udproute.go @@ -82,7 +82,7 @@ func NewUDPRouteManager( udpRouteLister: udpRouteInformer.Lister(), udpRouteListerSynced: udpRouteInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "gatewayClass"), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "udpRoute"), } manager.udpRouteInformer.Informer().AddEventHandler( diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go index 82857eb..58527af 100644 --- a/pkg/agent/manager/loadbalancer/loadbalancer.go +++ b/pkg/agent/manager/loadbalancer/loadbalancer.go @@ -71,6 +71,7 @@ const ( zoneSelAnnotation = "loxilb.io/zoneselect" prefLocalPodAnnotation = "loxilb.io/prefLocalPod" matchNodeLabelAnnotation = "loxilb.io/nodelabel" + enableTlsAnnotation = "loxilb.io/tls" MaxExternalSecondaryIPsNum = 4 ) @@ -111,6 +112,7 @@ type LbArgs struct { secIPs []string endpointIPs []string needPodEP bool + security int32 } type LbModelEnt struct { @@ -142,6 +144,7 @@ type LbCacheEntry struct { ProbeTimeo uint32 ProbeRetries int EpSelect api.EpSelect + Security int32 SecIPs []string LbServicePairs map[string]*LbServicePairEntry } @@ -367,11 +370,19 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error { prefLocal := false epSelect := api.LbSelRr matchNodeLabel := "" + enableTls := int32(0) if strings.Compare(*lbClassName, m.networkConfig.LoxilbLoadBalancerClass) != 0 && !needPodEP { return nil } + // Check for loxilb specific annotations - enableTlsAnnotation + if tls := svc.Annotations[enableTlsAnnotation]; tls != "" { + if tls == "true" { + enableTls = 1 + } + } + // Check for loxilb specific annotations - MatchNodeLabel if mnl := svc.Annotations[matchNodeLabelAnnotation]; mnl != "" { matchNodeLabel = mnl @@ -579,6 +590,7 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error { ProbeRetries: probeRetries, EpSelect: epSelect, Addr: addrType, + Security: enableTls, SecIPs: []string{}, LbServicePairs: make(map[string]*LbServicePairEntry), } @@ -843,6 +855,7 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error { probeTimeo: m.lbCache[cacheKey].ProbeTimeo, probeRetries: m.lbCache[cacheKey].ProbeRetries, sel: m.lbCache[cacheKey].EpSelect, + security: m.lbCache[cacheKey].Security, needPodEP: needPodEP, } lbArgs.secIPs = append(lbArgs.secIPs, m.lbCache[cacheKey].SecIPs...) @@ -1604,6 +1617,7 @@ func (m *Manager) makeLoxiLoadBalancerModel(lbArgs *LbArgs, svc *corev1.Service, ProbeTimeout: lbArgs.probeTimeo, ProbeRetries: int32(lbArgs.probeRetries), Sel: lbArgs.sel, + Security: lbArgs.security, Name: fmt.Sprintf("%s_%s", svc.Namespace, svc.Name), }, SecondaryIPs: loxiSecIPModelList, diff --git a/pkg/api/lb.go b/pkg/api/lb.go index 60c0aba..c73b299 100644 --- a/pkg/api/lb.go +++ b/pkg/api/lb.go @@ -71,6 +71,7 @@ type LoadBalancerService struct { ProbeResp string `json:"proberesp"` ProbeRetries int32 `json:"probeRetries,omitempty"` ProbeTimeout uint32 `json:"probeTimeout,omitempty"` + Security int32 `json:"security,omitempty"` Name string `json:"name,omitempty"` Oper LbOP `json:"oper,omitempty"` } From a3b90e524b6738c5e8cd3f7844aaff6ff06d8d52 Mon Sep 17 00:00:00 2001 From: backguynn Date: Fri, 5 Jul 2024 16:07:17 +0900 Subject: [PATCH 4/5] remove tls annotation --- pkg/agent/manager/loadbalancer/loadbalancer.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go index 58527af..c4543c6 100644 --- a/pkg/agent/manager/loadbalancer/loadbalancer.go +++ b/pkg/agent/manager/loadbalancer/loadbalancer.go @@ -370,19 +370,11 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error { prefLocal := false epSelect := api.LbSelRr matchNodeLabel := "" - enableTls := int32(0) if strings.Compare(*lbClassName, m.networkConfig.LoxilbLoadBalancerClass) != 0 && !needPodEP { return nil } - // Check for loxilb specific annotations - enableTlsAnnotation - if tls := svc.Annotations[enableTlsAnnotation]; tls != "" { - if tls == "true" { - enableTls = 1 - } - } - // Check for loxilb specific annotations - MatchNodeLabel if mnl := svc.Annotations[matchNodeLabelAnnotation]; mnl != "" { matchNodeLabel = mnl @@ -590,7 +582,6 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error { ProbeRetries: probeRetries, EpSelect: epSelect, Addr: addrType, - Security: enableTls, SecIPs: []string{}, LbServicePairs: make(map[string]*LbServicePairEntry), } From a5ebc3abdac6a74251d9536124a58f40854633c0 Mon Sep 17 00:00:00 2001 From: backguynn Date: Fri, 5 Jul 2024 16:12:52 +0900 Subject: [PATCH 5/5] remove tls annotation --- pkg/agent/manager/loadbalancer/loadbalancer.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go index c4543c6..82857eb 100644 --- a/pkg/agent/manager/loadbalancer/loadbalancer.go +++ b/pkg/agent/manager/loadbalancer/loadbalancer.go @@ -71,7 +71,6 @@ const ( zoneSelAnnotation = "loxilb.io/zoneselect" prefLocalPodAnnotation = "loxilb.io/prefLocalPod" matchNodeLabelAnnotation = "loxilb.io/nodelabel" - enableTlsAnnotation = "loxilb.io/tls" MaxExternalSecondaryIPsNum = 4 ) @@ -112,7 +111,6 @@ type LbArgs struct { secIPs []string endpointIPs []string needPodEP bool - security int32 } type LbModelEnt struct { @@ -144,7 +142,6 @@ type LbCacheEntry struct { ProbeTimeo uint32 ProbeRetries int EpSelect api.EpSelect - Security int32 SecIPs []string LbServicePairs map[string]*LbServicePairEntry } @@ -846,7 +843,6 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error { probeTimeo: m.lbCache[cacheKey].ProbeTimeo, probeRetries: m.lbCache[cacheKey].ProbeRetries, sel: m.lbCache[cacheKey].EpSelect, - security: m.lbCache[cacheKey].Security, needPodEP: needPodEP, } lbArgs.secIPs = append(lbArgs.secIPs, m.lbCache[cacheKey].SecIPs...) @@ -1608,7 +1604,6 @@ func (m *Manager) makeLoxiLoadBalancerModel(lbArgs *LbArgs, svc *corev1.Service, ProbeTimeout: lbArgs.probeTimeo, ProbeRetries: int32(lbArgs.probeRetries), Sel: lbArgs.sel, - Security: lbArgs.security, Name: fmt.Sprintf("%s_%s", svc.Namespace, svc.Name), }, SecondaryIPs: loxiSecIPModelList,