From 3ffc0cc3c01027348614db82e4445bd58b60ddff Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Fri, 19 Jan 2024 23:31:00 +0900
Subject: [PATCH] Fixed IPAM full issue in certain cases

---
 .../workloads/iperf-loxilb-daemonset.yaml     | 37 +++++++++++++++++++
 .../workloads/rabbitmq-loxilb-deployment.yaml | 22 +++++++++++
 .../manager/loadbalancer/loadbalancer.go      | 24 +++++-------
 pkg/ippool/ippool.go                          |  1 +
 4 files changed, 70 insertions(+), 14 deletions(-)
 create mode 100644 manifest/workloads/iperf-loxilb-daemonset.yaml
 create mode 100644 manifest/workloads/rabbitmq-loxilb-deployment.yaml

diff --git a/manifest/workloads/iperf-loxilb-daemonset.yaml b/manifest/workloads/iperf-loxilb-daemonset.yaml
new file mode 100644
index 0000000..89691ef
--- /dev/null
+++ b/manifest/workloads/iperf-loxilb-daemonset.yaml
@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: iperf-service
+  annotations:
+   loxilb.io/lbmode: "onearm"
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    app: iperf-app
+  ports:
+    - port: 55001
+      targetPort: 5001
+  type: LoadBalancer
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: iperf-daemonset
+  labels:
+    app: iperf-app
+spec:
+  selector:
+    matchLabels:
+      app: iperf-app
+  template:
+    metadata:
+      labels:
+        app: iperf-app
+    spec:
+      containers:
+      - name: iperf
+        image: eyes852/ubuntu-iperf-test:0.5
+        command: [ "iperf", "-s"]
+        ports:
+        - containerPort: 5001
diff --git a/manifest/workloads/rabbitmq-loxilb-deployment.yaml b/manifest/workloads/rabbitmq-loxilb-deployment.yaml
new file mode 100644
index 0000000..5255b70
--- /dev/null
+++ b/manifest/workloads/rabbitmq-loxilb-deployment.yaml
@@ -0,0 +1,22 @@
+apiVersion: rabbitmq.com/v1beta1
+kind: RabbitmqCluster
+metadata:
+  name: hello-world
+spec:
+  replicas: 3
+  rabbitmq:
+    additionalConfig: |
+      log.console.level = info
+      channel_max = 1700
+      default_user= guest
+      default_pass = guest
+      default_user_tags.administrator = true
+  service:
+    type: LoadBalancer
+  override:
+    service:
+      spec:
+        loadBalancerClass: loxilb.io/loxilb
+        externalTrafficPolicy: Local
+        ports:
+        - port: 5672
diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go
index 672d753..6242da8 100644
--- a/pkg/agent/manager/loadbalancer/loadbalancer.go
+++ b/pkg/agent/manager/loadbalancer/loadbalancer.go
@@ -686,12 +686,12 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error {
 		}
 		lbModelList = append(lbModelList, LbModelEnt{ingSvcPair.InRange, ingSvcPair.StaticIP, ingSvcPair.IdentIPAM, lbModel})
 
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-		defer cancel()
 		for _, client := range m.LoxiClients {
 			ch := make(chan error)
 			go func(c *api.LoxiClient, h chan error) {
 				var err error
+				ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+				defer cancel()
 				for _, lb := range lbModelList {
 					if err = c.LoadBalancer().Create(ctx, &lb.LbModel); err != nil {
 						if !strings.Contains(err.Error(), "exist") {
@@ -1014,22 +1014,18 @@ func (m *Manager) getIngressSvcPairs(service *corev1.Service, addrType string) (
 		for _, port := range service.Spec.Ports {
 			proto := strings.ToLower(string(port.Protocol))
 			portNum := port.Port
+			newIP, identIPAM = ipPool.GetNewIPAddr(cacheKey, uint32(portNum), proto)
 			if newIP == nil {
-				newIP, identIPAM = ipPool.GetNewIPAddr(cacheKey, uint32(portNum), proto)
-				if newIP == nil {
-					// This is a safety code in case the service has the same port.
-					for _, s := range sPairs {
-						if s.Port == portNum && s.Protocol == proto {
-							continue
-						}
+				// This is a safety code in case the service has the same port.
+				for _, s := range sPairs {
+					if s.Port == portNum && s.Protocol == proto {
+						continue
 					}
-					klog.Errorf("failed to generate external IP. IP Pool is full")
-					return nil, errors.New("failed to generate external IP. IP Pool is full"), hasExtIPAllocated
 				}
-				sp = SvcPair{newIP.String(), portNum, proto, true, false, identIPAM, port}
-			} else {
-				sp = SvcPair{newIP.String(), portNum, proto, false, true, identIPAM, port}
+				klog.Errorf("failed to generate external IP. IP Pool is full")
+				return nil, errors.New("failed to generate external IP. IP Pool is full"), hasExtIPAllocated
 			}
+			sp = SvcPair{newIP.String(), portNum, proto, true, false, identIPAM, port}
 			sPairs = append(sPairs, sp)
 		}
 	}
diff --git a/pkg/ippool/ippool.go b/pkg/ippool/ippool.go
index 7d471b0..96ed97d 100644
--- a/pkg/ippool/ippool.go
+++ b/pkg/ippool/ippool.go
@@ -70,6 +70,7 @@ func (i *IPPool) GetNewIPAddr(name string, sIdent uint32, proto string) (net.IP,
 			ipamIdent = tk.IPAMNoIdent
 			newIP, err := i.IPAlloc.AllocateNewIP(tk.IPClusterDefault, i.CIDR, ipamIdent)
 			if err != nil {
+				klog.Errorf("Allocate NewIP Failed %v:%s (%s)", sIdent, proto, err)
 				return nil, ipamIdent
 			}
 			return newIP, ipamIdent