add cilium support; fix nodeport issue; use DeployQueue to fix hangup issue.

cmd/controller/main.go

@@ -140,7 +140,7 @@ func main() {
 		// LeaderElectionReleaseOnCancel: true,
 	})
 	if err != nil {
-		setupLog.Error(err, "unable to start manager: %s", err.Error())
+		setupLog.Error(err, "unable to start manager: "+err.Error())
 		os.Exit(1)
 	}
 

deploy/README.md

@@ -1,6 +1,13 @@
 These two images cannot be downloaded in China and need to be processed separately.
 
+# for 0.5.1
 k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1
 gcr.io/k8s-staging-gateway-api/admission-server:v0.5.1
 
-Use docker save && docker load to setup the mentioned images.
+# for 0.6.0
+k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1
+gcr.io/k8s-staging-gateway-api/admission-server:v0.6.0
+
+Use `docker save & docker load` to setup the mentioned images for docker env.
+Use `ctr -n k8s.io image export & import` to setup the mentioned images for containerd env.
+*The images’ format is universal: docker save -> ctr -n k8s.io image import*

go.mod

@@ -3,7 +3,7 @@ module github.com/f5devcentral/bigip-kubernetes-gateway
 go 1.19
 
 require (
-	github.com/f5devcentral/f5-bigip-rest-go v1.0.7
+	github.com/f5devcentral/f5-bigip-rest-go v1.1.1
 	github.com/google/uuid v1.3.0
 	github.com/onsi/ginkgo/v2 v2.9.1
 	github.com/onsi/gomega v1.27.3

go.sum

@@ -68,8 +68,8 @@ github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMi
 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
 github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
-github.com/f5devcentral/f5-bigip-rest-go v1.0.7 h1:p8dxN4hQz3s/VIQXn1mfGuM7XKpCK10tORyso1+S8y8=
-github.com/f5devcentral/f5-bigip-rest-go v1.0.7/go.mod h1:gFO+eU59RmrjgZgyxJMPghtiP4+H0WVp5g5ChHKHDjA=
+github.com/f5devcentral/f5-bigip-rest-go v1.1.1 h1:MR4aVpL8rFw9wcS0c9DPImdM7VrKZtgQqyPxMpC/TYU=
+github.com/f5devcentral/f5-bigip-rest-go v1.1.1/go.mod h1:gFO+eU59RmrjgZgyxJMPghtiP4+H0WVp5g5ChHKHDjA=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=

internal/controllers/v1_controller.go

@@ -192,13 +192,13 @@ func handleDeletingEndpoints(ctx context.Context, req ctrl.Request) (ctrl.Result
 			return ctrl.Result{}, err
 		}
 
-		pkg.PendingDeploys <- deployer.DeployRequest{
+		pkg.PendingDeploys.Add(deployer.DeployRequest{
 			Meta:      fmt.Sprintf("deleting endpoints '%s'", req.NamespacedName.String()),
 			From:      &opcfgs,
 			To:        &npcfgs,
 			Partition: "cis-c-tenant",
 			Context:   ctx,
-		}
+		})
 
 	}
 
@@ -230,13 +230,13 @@ func handleUpsertingEndpoints(ctx context.Context, obj *v1.Endpoints) (ctrl.Resu
 			return ctrl.Result{}, err
 		}
 
-		pkg.PendingDeploys <- deployer.DeployRequest{
+		pkg.PendingDeploys.Add(deployer.DeployRequest{
 			Meta:      fmt.Sprintf("upserting endpoints '%s'", reqnsn),
 			From:      &opcfgs,
 			To:        &npcfgs,
 			Partition: "cis-c-tenant",
 			Context:   ctx,
-		}
+		})
 	}
 
 	return ctrl.Result{}, nil
@@ -265,13 +265,13 @@ func handleDeletingService(ctx context.Context, req ctrl.Request) (ctrl.Result,
 			return ctrl.Result{}, err
 		}
 
-		pkg.PendingDeploys <- deployer.DeployRequest{
+		pkg.PendingDeploys.Add(deployer.DeployRequest{
 			Meta:      fmt.Sprintf("deleting service '%s'", req.NamespacedName.String()),
 			From:      &opcfgs,
 			To:        &npcfgs,
 			Partition: "cis-c-tenant",
 			Context:   ctx,
-		}
+		})
 
 	}
 
@@ -304,13 +304,13 @@ func handleUpsertingService(ctx context.Context, obj *v1.Service) (ctrl.Result,
 			return ctrl.Result{}, err
 		}
 
-		pkg.PendingDeploys <- deployer.DeployRequest{
+		pkg.PendingDeploys.Add(deployer.DeployRequest{
 			Meta:      fmt.Sprintf("upserting service '%s'", reqnsn),
 			From:      &opcfgs,
 			To:        &npcfgs,
 			Partition: "cis-c-tenant",
 			Context:   ctx,
-		}
+		})
 	}
 
 	return ctrl.Result{}, nil

internal/k8s/k8s.go

@@ -25,17 +25,22 @@ func (ns *Nodes) Set(n *v1.Node) error {
 
 	node := K8Node{Name: n.Name}
 
-	// calico
-	if _, ok := n.Annotations["projectcalico.org/IPv4Address"]; ok {
-		ipmask := n.Annotations["projectcalico.org/IPv4Address"]
-		ipaddr := strings.Split(ipmask, "/")[0]
-		node = K8Node{
-			Name:    n.Name,
-			IpAddr:  ipaddr,
-			NetType: "calico-underlay",
-			MacAddr: "",
+	cnitype := detectCNIType(n)
+	switch cnitype {
+	case "cilium":
+		addrs := n.Status.Addresses
+		ipaddr := ""
+		for _, addr := range addrs {
+			if addr.Type == v1.NodeInternalIP {
+				ipaddr = addr.Address
+				break
+			}
 		}
-	} else {
+		node.Name = n.Name
+		node.IpAddr = ipaddr
+		node.NetType = ""
+		node.MacAddr = ipv4ToMac(ipaddr)
+	case "flannel":
 		// flannel v4
 		if _, ok := n.Annotations["flannel.alpha.coreos.com/backend-data"]; ok {
 			macStr := n.Annotations["flannel.alpha.coreos.com/backend-data"]
@@ -65,6 +70,17 @@ func (ns *Nodes) Set(n *v1.Node) error {
 				node.MacAddrV6 = v6["VtepMAC"].(string)
 			}
 		}
+	case "calico":
+		ipmask := n.Annotations["projectcalico.org/IPv4Address"]
+		ipaddr := strings.Split(ipmask, "/")[0]
+		node = K8Node{
+			Name:    n.Name,
+			IpAddr:  ipaddr,
+			NetType: "calico-underlay",
+			MacAddr: "",
+		}
+	default:
+		return fmt.Errorf("unknown CNI type: %s for node %s", cnitype, n.Name)
 	}
 
 	NodeCache.mutex <- true

internal/k8s/utils.go

@@ -2,6 +2,8 @@ package k8s
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/f5devcentral/f5-bigip-rest-go/utils"
 	v1 "k8s.io/api/core/v1"
@@ -48,14 +50,14 @@ func FormatMembersFromServiceEndpoints(svc *v1.Service, eps *v1.Endpoints) ([]Sv
 					}
 					if k8no := NodeCache.Get(*addr.NodeName); k8no == nil {
 						return []SvcEpsMember{}, utils.RetryErrorf("%s not found yet", *addr.NodeName)
-					} else {
+					} else if k8no.NetType == "vxlan" {
 						if utils.IsIpv6(addr.IP) {
 							member.MacAddr = k8no.MacAddrV6
 						} else {
 							member.MacAddr = k8no.MacAddr
 						}
-						members = append(members, member)
 					}
+					members = append(members, member)
 				}
 			}
 		}
@@ -69,3 +71,36 @@ func FormatMembersFromServiceEndpoints(svc *v1.Service, eps *v1.Endpoints) ([]Sv
 
 	return members, nil
 }
+
+func detectCNIType(node *v1.Node) string {
+	kind := "unknown"
+
+	for _, c := range node.Status.Conditions {
+		if c.Reason == "CiliumIsUp" {
+			kind = "cilium"
+			break
+		}
+		if c.Reason == "CalicoIsUp" {
+			kind = "calico"
+			break
+		}
+		if c.Reason == "FlannelIsUp" {
+			kind = "flannel"
+			break
+		}
+	}
+	return kind
+}
+
+// Convert an IPV4 string to a fake MAC address.
+func ipv4ToMac(addr string) string {
+	ip := strings.Split(addr, ".")
+	if len(ip) != 4 {
+		return ""
+	}
+	var intIP [4]int
+	for i, val := range ip {
+		intIP[i], _ = strconv.Atoi(val)
+	}
+	return fmt.Sprintf("0a:0a:%02x:%02x:%02x:%02x", intIP[0], intIP[1], intIP[2], intIP[3])
+}

internal/pkg/utils.go

@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"strings"
 
+	f5_bigip "github.com/f5devcentral/f5-bigip-rest-go/bigip"
 	"github.com/f5devcentral/f5-bigip-rest-go/deployer"
 	"github.com/f5devcentral/f5-bigip-rest-go/utils"
 	v1 "k8s.io/api/core/v1"
@@ -217,6 +218,15 @@ func DeployForEvent(ctx context.Context, impactedClasses []string, apply func()
 		}
 	}
 
+	// TODO: fix the issue:
+	//	2023/06/19 09:26:49.824036 [ERROR] [cd7c411d-e392-424f-8a76-be055f1286d2] \
+	//	failed to deploy partition cis-c-tenant: 400, {"code":400,"message":"0107082a:3: \
+	//	All objects must be removed from a partition (cis-c-tenant) before the partition may be removed, type ID (973)","errorStack":[],"apiError":3}
+
+	// TODO: fix the issue:
+	// 2023/06/19 09:17:39.572853 [ERROR] [a763bd16-498a-415a-89a3-f5fdf2aa5adf] \
+	//	failed to do deployment to https://10.250.15.109:443: 400, {"code":400,"message":"transaction failed:01070110:3: \
+	//	Node address '/cis-c-tenant/10.250.16.103' is referenced by a member of pool '/cis-c-tenant/default.dev-service'.","errorStack":[],"apiError":2}
 	drs["cis-c-tenant"] = &deployer.DeployRequest{
 		Meta:      fmt.Sprintf("Updating pools for event %s", meta),
 		From:      &opcfgs,
@@ -226,7 +236,7 @@ func DeployForEvent(ctx context.Context, impactedClasses []string, apply func()
 	}
 
 	for _, dr := range drs {
-		PendingDeploys <- *dr
+		PendingDeploys.Add(*dr)
 	}
 
 	return nil
@@ -291,3 +301,67 @@ func validateSecretType(group *gatewayv1beta1.Group, kind *gatewayv1beta1.Kind)
 	}
 	return nil
 }
+
+// purgeCommonNodes tries to remove  nodes from Common if no reference.
+func purgeCommonNodes(ctx context.Context, ombs []interface{}) {
+	for _, bp := range BIGIPs {
+		bc := f5_bigip.BIGIPContext{Context: ctx, BIGIP: *bp}
+		slog := utils.LogFromContext(ctx)
+
+		for _, m := range ombs {
+			partition := m.(map[string]interface{})["partition"].(string)
+			if partition != "Common" {
+				continue
+			}
+			addr := m.(map[string]interface{})["address"].(string)
+			err := bc.Delete("ltm/node", addr, "Common", "")
+			if err != nil && !strings.Contains(err.Error(), "is referenced by a member of pool") {
+				slog.Warnf("cannot delete node %s: %s", addr, err.Error())
+			}
+		}
+	}
+}
+
+// // splitByPartition split the cfgs into a map of which keys are partitions
+// func splitByPartition(ctx context.Context, cfgs map[string]interface{}) map[string]interface{} {
+// 	partitions := map[string]map[string]map[string]interface{}{}
+// 	for fstr, fv := range cfgs {
+// 		for rstr, rv := range fv.(map[string]interface{}) {
+// 			pstr := "unknown"
+// 			if p, f := rv.(map[string]interface{})["partition"]; f {
+// 				pstr = p.(string)
+// 			}
+
+// 			if _, pok := partitions[pstr]; !pok {
+// 				partitions[pstr] = map[string]map[string]interface{}{}
+
+// 			}
+// 			if _, fok := partitions[pstr][fstr]; !fok {
+// 				partitions[pstr][fstr] = map[string]interface{}{}
+// 			}
+// 			partitions[pstr][fstr][rstr] = rv
+// 		}
+// 	}
+// 	rlt := map[string]interface{}{}
+// 	for p, v := range partitions {
+// 		rlt[p] = v
+// 	}
+// 	return rlt
+// }
+
+// filterCommonResources filter the 'Common' resources from cfgs
+func filterCommonResources(cfgs map[string]interface{}) map[string]interface{} {
+	rlt := map[string]interface{}{}
+	for fstr, fv := range cfgs {
+		if _, ok := rlt[fstr]; !ok {
+			rlt[fstr] = map[string]interface{}{}
+		}
+		for rstr, rv := range fv.(map[string]interface{}) {
+			if p, f := rv.(map[string]interface{})["partition"]; f && p == "Common" {
+				rlt[fstr].(map[string]interface{})[rstr] = rv
+				delete(cfgs[fstr].(map[string]interface{}), rstr)
+			}
+		}
+	}
+	return rlt
+}