Merge branch 'main' of github.com:backguynn/test-kube-loxilb

loxilb-io · Dec 22, 2023 · 8e17844 · 8e17844
2 parents 506e84a + dbb8007
commit 8e17844
Show file tree

Hide file tree

Showing 8 changed files with 256 additions and 33 deletions.
diff --git a/cmd/loxilb-agent/agent.go b/cmd/loxilb-agent/agent.go
@@ -30,7 +30,6 @@ import (
 	"github.com/loxilb-io/kube-loxilb/pkg/k8s"
 	"github.com/loxilb-io/kube-loxilb/pkg/log"
 
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
 	"k8s.io/klog/v2"
 
@@ -155,10 +154,12 @@ func run(o *Options) error {
 	loxiLBLiveCh := make(chan *api.LoxiClient, 50)
 	loxiLBPurgeCh := make(chan *api.LoxiClient, 5)
 	loxiLBSelMasterEvent := make(chan bool)
+	loxiLBDeadCh := make(chan struct{}, 64)
+	ticker := time.NewTicker(20 * time.Second)
 
 	if len(networkConfig.LoxilbURLs) > 0 {
 		for _, lbURL := range networkConfig.LoxilbURLs {
-			loxilbClient, err := api.NewLoxiClient(lbURL, loxiLBLiveCh, false)
+			loxilbClient, err := api.NewLoxiClient(lbURL, loxiLBLiveCh, loxiLBDeadCh, false)
 			if err != nil {
 				return err
 			}
@@ -178,17 +179,28 @@ func run(o *Options) error {
 		informerFactory,
 	)
 
-	go wait.Until(func() {
-		if len(networkConfig.LoxilbURLs) <= 0 {
-			lbManager.DiscoverLoxiLBServices(loxiLBLiveCh, loxiLBPurgeCh)
-		}
-		lbManager.DiscoverLoxiLBPeerServices(loxiLBLiveCh, loxiLBPurgeCh)
-
-		if networkConfig.SetRoles != "" {
-			lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+	go func() {
+		for {
+			select {
+			case <-loxiLBDeadCh:
+				if networkConfig.SetRoles != "" {
+					klog.Infof("Running select-roles")
+					lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+				}
+			case <-ticker.C:
+				if len(networkConfig.LoxilbURLs) <= 0 {
+					lbManager.DiscoverLoxiLBServices(loxiLBLiveCh, loxiLBDeadCh, loxiLBPurgeCh)
+				}
+				lbManager.DiscoverLoxiLBPeerServices(loxiLBLiveCh, loxiLBDeadCh, loxiLBPurgeCh)
+
+				if networkConfig.SetRoles != "" {
+					lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+				}
+			case <-stopCh:
+				return
+			}
 		}
-	}, time.Second*20, stopCh)
-
+	}()
 	log.StartLogFileNumberMonitor(stopCh)
 	informerFactory.Start(stopCh)
 

diff --git a/cmd/loxilb-agent/main.go b/cmd/loxilb-agent/main.go
@@ -37,7 +37,7 @@ func Execute() {
 		Long:  "loxilb-k8s",
 	}
 
-	client, err := api.NewLoxiClient("http://127.0.0.1:11111", nil, false)
+	client, err := api.NewLoxiClient("http://127.0.0.1:11111", nil, nil, false)
 	if err != nil {
 		return
 	}

diff --git a/manifest/ext-peer-cluster/kube-loxilb.yaml b/manifest/ext-peer-cluster/kube-loxilb.yaml
@@ -0,0 +1,130 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - services/status
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+      - update
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-loxilb
+subjects:
+  - kind: ServiceAccount
+    name: kube-loxilb
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+  labels:
+    app: kube-loxilb-app
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kube-loxilb-app
+  template:
+    metadata:
+      labels:
+        app: kube-loxilb-app
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      tolerations:
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-loxilb
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: kube-loxilb
+        image: ghcr.io/loxilb-io/kube-loxilb:latest
+        imagePullPolicy: Always
+        command:
+        - /bin/kube-loxilb
+        args:
+        - --loxiURL=http://192.168.80.10:11111
+        - --externalCIDR=123.123.123.1/24
+        #- --externalSecondaryCIDRs=124.124.124.1/24,125.125.125.1/24
+        - --setBGP=64512
+        - --listenBGPPort=1791
+        - --setRoles=0.0.0.0
+        #- --monitor
+        #- --extBGPPeers=50.50.50.1:65101,51.51.51.1:65102
+        #- --setLBMode=1
+        #- --config=/opt/loxilb/agent/kube-loxilb.conf
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW"]
diff --git a/manifest/ext-peer-cluster/loxilb-peer.yaml b/manifest/ext-peer-cluster/loxilb-peer.yaml
@@ -0,0 +1,64 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: loxilb-peer
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: loxilb-peer-app
+  template:
+    metadata:
+      name: loxilb-peer
+      labels:
+        app: loxilb-peer-app
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      # Mark the pod as a critical add-on for rescheduling.
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: Exists
+      containers:
+      - name: loxilb-peer-app
+        image: "ghcr.io/loxilb-io/loxilb:latest"
+        command: [ "/root/loxilb-io/loxilb/loxilb", "--peer" ]
+        ports:
+        - containerPort: 11111
+        - containerPort: 1791
+        - containerPort: 50051
+        securityContext:
+          privileged: true
+          capabilities:
+            add:
+              - SYS_ADMIN
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loxilb-peer-service
+  namespace: kube-system
+spec:
+  clusterIP: None
+  selector:
+    app: loxilb-peer-app
+  ports:
+  - name: loxilb-peer-app
+    port: 11111
+    targetPort: 11111
+    protocol: TCP
+  - name: loxilb-peer-bgp
+    port: 1791
+    targetPort: 1791
+    protocol: TCP
+  - name: loxilb-peer-gobgp
+    port: 50051
+    targetPort: 50051
+    protocol: TCP
diff --git a/manifest/in-cluster/loxilb-peer.yml → manifest/in-cluster/loxilb-peer.yaml b/manifest/in-cluster/loxilb-peer.yml → manifest/in-cluster/loxilb-peer.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -60,5 +61,3 @@ spec:
     port: 50051
     targetPort: 50051
     protocol: TCP
-
-
diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go
@@ -761,16 +761,17 @@ func (m *Manager) deleteLoadBalancer(ns, name string) error {
 			}(loxiClient, ch)
 		}
 
+		var err error
 		isError := true
 		for _, errCh := range errChList {
-			err := <-errCh
+			err = <-errCh
 			if err == nil {
 				isError = false
 				break
 			}
 		}
 		if isError {
-			return fmt.Errorf("failed to delete loxiLB LoadBalancer")
+			return fmt.Errorf("failed to delete loxiLB LoadBalancer. err: %v", err)
 		}
 		if lb.inRange {
 			ipPool.ReturnIPAddr(lb.LbModel.Service.ExternalIP, lb.IdentIPAM)
@@ -1244,7 +1245,7 @@ func (m *Manager) addIngress(service *corev1.Service, newIP net.IP) {
 		append(service.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{IP: newIP.String()})
 }
 
-func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBPurgeCh chan *api.LoxiClient) {
+func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBDeadCh chan struct{}, loxiLBPurgeCh chan *api.LoxiClient) {
 	var tmploxilbClients []*api.LoxiClient
 	// DNS lookup (not used now)
 	// ips, err := net.LookupIP("loxilb-lb-service")
@@ -1271,7 +1272,7 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox
 			}
 		}
 		if !found {
-			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, false)
+			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, loxiLBDeadCh, false)
 			if err2 != nil {
 				continue
 			}
@@ -1294,10 +1295,12 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox
 	m.LoxiClients = tmp
 }
 
-func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBPurgeCh chan *api.LoxiClient) {
+func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBDeadCh chan struct{}, loxiLBPurgeCh chan *api.LoxiClient) {
 	var tmploxilbPeerClients []*api.LoxiClient
 	ips, err := k8s.GetServiceEndPoints(m.kubeClient, "loxilb-peer-service", "kube-system")
-	klog.Infof("loxilb-peer-service end-points:  %v", ips)
+	if len(ips) > 0 {
+		klog.Infof("loxilb-peer-service end-points:  %v", ips)
+	}
 	if err != nil {
 		ips = []net.IP{}
 	}
@@ -1319,7 +1322,7 @@ func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient,
 			}
 		}
 		if !found {
-			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, true)
+			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, loxiLBDeadCh, true)
 			if err2 != nil {
 				continue
 			}
@@ -1366,7 +1369,7 @@ func (m *Manager) SelectLoxiLBRoles(sendSigCh bool, loxiLBSelMasterEvent chan bo
 				if v.IsAlive {
 					v.MasterLB = true
 					selMaster = true
-					klog.Infof("loxilb-peer(%v) set-role master", v.Url)
+					klog.Infof("loxilb-lb(%v) set-role master", v.Url)
 				}
 			}
 			if selMaster {
@@ -1483,9 +1486,11 @@ loop:
 							bgpPeers = append(bgpPeers, lpc)
 						}
 					}
-					for _, lc := range m.LoxiClients {
-						if aliveClient.Host != lc.Host {
-							bgpPeers = append(bgpPeers, lc)
+					if len(m.networkConfig.LoxilbURLs) <= 0 {
+						for _, lc := range m.LoxiClients {
+							if aliveClient.Host != lc.Host {
+								bgpPeers = append(bgpPeers, lc)
+							}
 						}
 					}
 				}