add base CICD

backguynn · Mar 29, 2024 · 8e44f1c · 8e44f1c
1 parent a168afa
commit 8e44f1c
Show file tree

Hide file tree

Showing 4 changed files with 175 additions and 3 deletions.
diff --git a/.github/workflows/kube-loxilb-base-sanity.yml b/.github/workflows/kube-loxilb-base-sanity.yml
@@ -0,0 +1,136 @@
+name: check kube-loxilb base sanity
+on: [push]
+jobs:
+  test-kube-loxilb:
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+    steps:
+      - name: install k3d
+        run: wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+
+      - name: make k3d single node
+        run: k3d cluster create test-cluster --k3s-arg "--disable=traefik@server:*" --k3s-arg "--disable=servicelb@server:*" --k3s-arg "--kubelet-arg=cloud-provider=external@server:*"
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.19.0'
+
+      - name: build kube-loxilb's new image
+        run: make docker 
+
+      - name: install gateway api
+        run: kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml
+
+      - name: deploy kube-loxilb & loxilb & gateway resources
+        run: |
+          kubectl apply -f manifest/gateway-api/kube-loxilb.yaml
+          kubectl apply -f manifest/gateway-api/loxilb.yaml
+          kubectl apply -f manifest/gateway-api/gatewayclass.yaml
+          kubectl apply -f manifest/gateway-api/gateway.yaml
+          kubectl apply -f manifest/gateway-api/tcpRoute.yaml
+          kubectl apply -f manifest/gateway-api/udpRoute.yaml
+
+      - name: wait to deploy kube-loxilb pod
+        id: deploy_kube_loxilb
+        run: |
+          kubectl -n kube-system wait --for=condition=ready pod -l app=kube-loxilb-app --timeout=300s
+          KUBE_LOXILB=`kubectl get pods -A -o json | jq -r '.items[] | select(.metadata.name | test("kube-loxilb")).metadata.name'`
+          echo "KUBE_LOXILB=$KUBE_LOXILB" >> $GITHUB_OUTPUT
+
+      - if: failure()
+        run: |
+          kubectl -n kube-system get pods
+          exit 1
+          
+      - name: Sleep for 60 seconds for kube-loxilb
+        run: sleep 60s
+
+      - name: check status of gateway resources
+        id: get_results_json
+        run: |
+          GC_PASSED=$(kubectl get gatewayclass -o json | jq -r '.items[0].status.conditions[0] | select(.type | test("Accepted")).status')
+          echo "GC_PASSED=$GC_PASSED" >> $GITHUB_OUTPUT
+          GW_PASSED=$(kubectl get gateway -o json | jq -r '.items[0].status.conditions[] | select(.type | test("Programmed")).status')
+          echo "GW_PASSED=$GW_PASSED" >> $GITHUB_OUTPUT
+          TCPROUTE_PASSED=$(kubectl get svc tcproute-lb-service -o json | jq -r '.status.loadBalancer.ingress[0].hostname')
+          echo "TCPROUTE_PASSED=$TCPROUTE_PASSED" >> $GITHUB_OUTPUT
+          UDPROUTE_PASSED=$(kubectl get svc udproute-lb-service -o json | jq -r '.status.loadBalancer.ingress[0].hostname')
+          echo "UDPROUTE_PASSED=$UDPROUTE_PASSED" >> $GITHUB_OUTPUT
+
+      - if: ${{ steps.get_results_json.outputs.GC_PASSED != 'True' }}
+        run: |
+          echo "gatewayClass passed: $steps.get_results_json.outputs.GC_PASSED"
+          kubectl get gatewayclass
+          kubectl -n kube-system logs "${{ steps.deploy_kube_loxilb.outputs.KUBE_LOXILB }}"
+          exit 1
+
+      - if: ${{ steps.get_results_json.outputs.GW_PASSED != 'True' }}
+        run: |
+          echo "gateway passed: $steps.get_results_json.outputs.GW_PASSED"
+          kubectl get gateway
+          kubectl -n kube-system logs "${{ steps.deploy_kube_loxilb.outputs.KUBE_LOXILB }}"
+          exit 1
+
+      - if: ${{ steps.get_results_json.outputs.TCPROUTE_PASSED != 'llb-123.123.123.1' }}
+        run: |
+          echo "tcpRoute external IP: $steps.get_results_json.outputs.TCPROUTE_PASSED"
+          kubectl get tcproute
+          kubectl get svc tcproute-lb-service -o wide
+          kubectl -n kube-system logs "${{ steps.deploy_kube_loxilb.outputs.KUBE_LOXILB }}"
+          exit 1
+
+      - if: ${{ steps.get_results_json.outputs.UDPROUTE_PASSED != 'llb-123.123.123.1' }}
+        run: |
+          echo "udpRoute external IP: $steps.get_results_json.outputs.UDPROUTE_PASSED"
+          kubectl get udproute
+          kubectl get svc udproute-lb-service -o wide
+          kubectl -n kube-system logs "${{ steps.deploy_kube_loxilb.outputs.KUBE_LOXILB }}"
+          exit 1
+
+      - name: deploy services & pod resources
+        run: |
+          kubectl apply -f manifest/workloads/iperf.yaml
+          kubectl apply -f manifest/workloads/sctp-no-secips.yml
+          kubectl apply -f manifest/workloads/udp-echo-svc-lb.yml
+
+      - name: Sleep for 60 seconds for wait to deploy pod
+        run: sleep 60s
+
+      - name: check status of service resources
+        id: get_svc_results_json
+        run: |
+          IPERF_PASSED=$(kubectl get svc iperf-service -o json | jq -r '.status.loadBalancer.ingress[0].hostname')
+          echo "IPERF_PASSED=$IPERF_PASSED" >> $GITHUB_OUTPUT
+          SCTP_PASSED=$(kubectl get svc sctp-lb1 -o json | jq -r '.status.loadBalancer.ingress[0].hostname')
+          echo "SCTP_PASSED=$SCTP_PASSED" >> $GITHUB_OUTPUT
+          UDP_PASSED=$(kubectl get svc udp-lb1 -o json | jq -r '.status.loadBalancer.ingress[0].hostname')
+          echo "UDP_PASSED=$UDP_PASSED" >> $GITHUB_OUTPUT
+
+      - if: ${{ steps.get_svc_results_json.outputs.IPERF_PASSED != 'llb-123.123.123.1' }}
+        run: |
+          echo "iperf-service external IP: $steps.get_results_json.outputs.IPERF_PASSED"
+          kubectl get svc -o wide
+          kubectl -n kube-system logs "${{ steps.deploy_kube_loxilb.outputs.KUBE_LOXILB }}"
+          exit 1
+      - if: ${{ steps.get_svc_results_json.outputs.SCTP_PASSED != 'llb-123.123.123.1' }}
+        run: |
+          echo "iperf-service external IP: $steps.get_results_json.outputs.SCTP_PASSED"
+          kubectl get svc -o wide
+          kubectl -n kube-system logs "${{ steps.deploy_kube_loxilb.outputs.KUBE_LOXILB }}"
+          exit 1
+
+      - if: ${{ steps.get_svc_results_json.outputs.UDP_PASSED != 'llb-123.123.123.1' }}
+        run: |
+          echo "iperf-service external IP: $steps.get_results_json.outputs.UDP_PASSED"
+          kubectl get svc -o wide
+          kubectl -n kube-system logs "${{ steps.deploy_kube_loxilb.outputs.KUBE_LOXILB }}"
+          exit 1
+
+  test-k3s-base-sanity:
+    needs: test-kube-loxilb
+    uses: loxilb-io/loxilb/.github/workflows/__k3s-base-sanity-module.yml@main
diff --git a/manifest/gateway-api/kube-loxilb.yaml b/manifest/gateway-api/kube-loxilb.yaml
@@ -114,7 +114,7 @@ spec:
       terminationGracePeriodSeconds: 0
       containers:
       - name: kube-loxilb
-        image: ghcr.io/loxilb-io/kube-loxilb:gateway-debug
+        image: ghcr.io/loxilb-io/kube-loxilb:latest
         imagePullPolicy: Always
         command:
         - /bin/kube-loxilb
@@ -127,6 +127,7 @@ spec:
         - --listenBGPPort=1791
         - --setRoles=0.0.0.0
         - --gatewayAPI
+        - --v=4
         #- --extBGPPeers=50.50.50.1:65101,51.51.51.1:65102
         #- --setLBMode=1
         #- --config=/opt/loxilb/agent/kube-loxilb.conf

diff --git a/manifest/workloads/sctp-no-secips.yml b/manifest/workloads/sctp-no-secips.yml
@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sctp-lb1
+spec:
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    what: sctp-test
+  ports:
+    - port: 55002
+      protocol: SCTP
+      targetPort: 9999
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sctp-test
+  labels:
+    what: sctp-test
+spec:
+  terminationGracePeriodSeconds: 0
+  containers:
+    - name: sctp-test
+      image: aweimeow/sctp-server
+      command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"]
+      ports:
+        - containerPort: 9999
diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go
@@ -161,7 +161,7 @@ type SvcPair struct {
 }
 
 func (s SvcPair) String() string {
-	return fmt.Sprintf("  IPString: %s\n  Port: %d\n  Protocol: %s\n  InRange: %v\n  StaticIP: %v\n  IdentIPAM: %s\n  IPAllocd:  %v\n  K8sSvcPort: %v\n",
+	return fmt.Sprintf("\n  IPString: %s\n  Port: %d\n  Protocol: %s\n  InRange: %v\n  StaticIP: %v\n  IdentIPAM: %s\n  IPAllocd:  %v\n  K8sSvcPort: %v\n",
 		s.IPString, s.Port, s.Protocol, s.InRange, s.StaticIP, s.IdentIPAM, s.IPAllocd, s.K8sSvcPort,
 	)
 }
@@ -516,6 +516,8 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error {
 
 	endpointIPs, err := m.getEndpoints(svc, needPodEP, addrType)
 	if err != nil {
+		klog.Errorf("getEndpoints return error.")
+		klog.V(4).Infof("endpointIPs: %v", endpointIPs)
 		return err
 	}
 
@@ -575,7 +577,10 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error {
 				ipPool = m.ExternalIP6Pool
 				sipPools = m.ExtSecondaryIP6Pools
 			}
-			klog.Infof("deallocateOnFailure defer function called")
+			klog.Infof("deallocateOnFailure defer function called by service %s", svc.Name)
+			klog.V(4).Infof("error: %v", err)
+			klog.V(4).Infof("ingSvcPairs: %v", ingSvcPairs)
+			klog.V(4).Infof("hasExistingEIP: %v", hasExistingEIP)
 			for i, sp := range ingSvcPairs {
 				if sp.InRange && sp.IPAllocd {
 					klog.Infof("Returning ip %s to free pool", sp.IPString)
@@ -772,6 +777,7 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error {
 	}
 
 	if !update {
+		// TODO: Some cloud providers(e.g: K3d) delete external IPs assigned by kube-loxilb, so you can reach this syntax:
 		if !hasExistingEIP {
 			retIPAMOnErr = true
 		}
@@ -882,6 +888,7 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error {
 func (m *Manager) updateService(old, new *corev1.Service) error {
 	if !reflect.DeepEqual(old.Status, new.Status) {
 		_, err := m.kubeClient.CoreV1().Services(new.Namespace).UpdateStatus(context.TODO(), new, metav1.UpdateOptions{})
+		klog.V(4).Infof("service %s is updated status: %v", new.Name, new.Status.LoadBalancer.Ingress)
 		if err != nil {
 			klog.Errorf("failed to update service %s.status. err: %v", new.Name, err)
 			return err