diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 33c490e..bd7874b 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -140,7 +140,7 @@ func main() { // LeaderElectionReleaseOnCancel: true, }) if err != nil { - setupLog.Error(err, "unable to start manager: %s", err.Error()) + setupLog.Error(err, "unable to start manager: "+err.Error()) os.Exit(1) } diff --git a/deploy/README.md b/deploy/README.md index 128e234..cae8c34 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,6 +1,13 @@ These two images cannot be downloaded in China and need to be processed separately. +# for 0.5.1 k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1 gcr.io/k8s-staging-gateway-api/admission-server:v0.5.1 -Use docker save && docker load to setup the mentioned images. \ No newline at end of file +# for 0.6.0 +k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1 +gcr.io/k8s-staging-gateway-api/admission-server:v0.6.0 + +Use `docker save & docker load` to setup the mentioned images for docker env. +Use `ctr -n k8s.io image export & import` to setup the mentioned images for containerd env. +*The images’ format is universal: docker save -> ctr -n k8s.io image import* \ No newline at end of file diff --git a/go.mod b/go.mod index fe5c578..a87e2d5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/f5devcentral/bigip-kubernetes-gateway go 1.19 require ( - github.com/f5devcentral/f5-bigip-rest-go v1.0.7 + github.com/f5devcentral/f5-bigip-rest-go v1.1.1 github.com/google/uuid v1.3.0 github.com/onsi/ginkgo/v2 v2.9.1 github.com/onsi/gomega v1.27.3 diff --git a/go.sum b/go.sum index 39353d0..979cb4d 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMi github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/f5devcentral/f5-bigip-rest-go v1.0.7 h1:p8dxN4hQz3s/VIQXn1mfGuM7XKpCK10tORyso1+S8y8= -github.com/f5devcentral/f5-bigip-rest-go v1.0.7/go.mod h1:gFO+eU59RmrjgZgyxJMPghtiP4+H0WVp5g5ChHKHDjA= +github.com/f5devcentral/f5-bigip-rest-go v1.1.1 h1:MR4aVpL8rFw9wcS0c9DPImdM7VrKZtgQqyPxMpC/TYU= +github.com/f5devcentral/f5-bigip-rest-go v1.1.1/go.mod h1:gFO+eU59RmrjgZgyxJMPghtiP4+H0WVp5g5ChHKHDjA= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= diff --git a/internal/controllers/v1_controller.go b/internal/controllers/v1_controller.go index d0c34e8..07b73e8 100644 --- a/internal/controllers/v1_controller.go +++ b/internal/controllers/v1_controller.go @@ -192,13 +192,13 @@ func handleDeletingEndpoints(ctx context.Context, req ctrl.Request) (ctrl.Result return ctrl.Result{}, err } - pkg.PendingDeploys <- deployer.DeployRequest{ + pkg.PendingDeploys.Add(deployer.DeployRequest{ Meta: fmt.Sprintf("deleting endpoints '%s'", req.NamespacedName.String()), From: &opcfgs, To: &npcfgs, Partition: "cis-c-tenant", Context: ctx, - } + }) } @@ -230,13 +230,13 @@ func handleUpsertingEndpoints(ctx context.Context, obj *v1.Endpoints) (ctrl.Resu return ctrl.Result{}, err } - pkg.PendingDeploys <- deployer.DeployRequest{ + pkg.PendingDeploys.Add(deployer.DeployRequest{ Meta: fmt.Sprintf("upserting endpoints '%s'", reqnsn), From: &opcfgs, To: &npcfgs, Partition: "cis-c-tenant", Context: ctx, - } + }) } return ctrl.Result{}, nil @@ -265,13 +265,13 @@ func handleDeletingService(ctx context.Context, req ctrl.Request) (ctrl.Result, return ctrl.Result{}, err } - pkg.PendingDeploys <- deployer.DeployRequest{ + pkg.PendingDeploys.Add(deployer.DeployRequest{ Meta: fmt.Sprintf("deleting service '%s'", req.NamespacedName.String()), From: &opcfgs, To: &npcfgs, Partition: "cis-c-tenant", Context: ctx, - } + }) } @@ -304,13 +304,13 @@ func handleUpsertingService(ctx context.Context, obj *v1.Service) (ctrl.Result, return ctrl.Result{}, err } - pkg.PendingDeploys <- deployer.DeployRequest{ + pkg.PendingDeploys.Add(deployer.DeployRequest{ Meta: fmt.Sprintf("upserting service '%s'", reqnsn), From: &opcfgs, To: &npcfgs, Partition: "cis-c-tenant", Context: ctx, - } + }) } return ctrl.Result{}, nil diff --git a/internal/k8s/k8s.go b/internal/k8s/k8s.go index fc8d875..7fc81de 100644 --- a/internal/k8s/k8s.go +++ b/internal/k8s/k8s.go @@ -25,17 +25,22 @@ func (ns *Nodes) Set(n *v1.Node) error { node := K8Node{Name: n.Name} - // calico - if _, ok := n.Annotations["projectcalico.org/IPv4Address"]; ok { - ipmask := n.Annotations["projectcalico.org/IPv4Address"] - ipaddr := strings.Split(ipmask, "/")[0] - node = K8Node{ - Name: n.Name, - IpAddr: ipaddr, - NetType: "calico-underlay", - MacAddr: "", + cnitype := detectCNIType(n) + switch cnitype { + case "cilium": + addrs := n.Status.Addresses + ipaddr := "" + for _, addr := range addrs { + if addr.Type == v1.NodeInternalIP { + ipaddr = addr.Address + break + } } - } else { + node.Name = n.Name + node.IpAddr = ipaddr + node.NetType = "" + node.MacAddr = ipv4ToMac(ipaddr) + case "flannel": // flannel v4 if _, ok := n.Annotations["flannel.alpha.coreos.com/backend-data"]; ok { macStr := n.Annotations["flannel.alpha.coreos.com/backend-data"] @@ -65,6 +70,17 @@ func (ns *Nodes) Set(n *v1.Node) error { node.MacAddrV6 = v6["VtepMAC"].(string) } } + case "calico": + ipmask := n.Annotations["projectcalico.org/IPv4Address"] + ipaddr := strings.Split(ipmask, "/")[0] + node = K8Node{ + Name: n.Name, + IpAddr: ipaddr, + NetType: "calico-underlay", + MacAddr: "", + } + default: + return fmt.Errorf("unknown CNI type: %s for node %s", cnitype, n.Name) } NodeCache.mutex <- true diff --git a/internal/k8s/utils.go b/internal/k8s/utils.go index 1969efd..508d240 100644 --- a/internal/k8s/utils.go +++ b/internal/k8s/utils.go @@ -2,6 +2,8 @@ package k8s import ( "fmt" + "strconv" + "strings" "github.com/f5devcentral/f5-bigip-rest-go/utils" v1 "k8s.io/api/core/v1" @@ -48,14 +50,14 @@ func FormatMembersFromServiceEndpoints(svc *v1.Service, eps *v1.Endpoints) ([]Sv } if k8no := NodeCache.Get(*addr.NodeName); k8no == nil { return []SvcEpsMember{}, utils.RetryErrorf("%s not found yet", *addr.NodeName) - } else { + } else if k8no.NetType == "vxlan" { if utils.IsIpv6(addr.IP) { member.MacAddr = k8no.MacAddrV6 } else { member.MacAddr = k8no.MacAddr } - members = append(members, member) } + members = append(members, member) } } } @@ -69,3 +71,36 @@ func FormatMembersFromServiceEndpoints(svc *v1.Service, eps *v1.Endpoints) ([]Sv return members, nil } + +func detectCNIType(node *v1.Node) string { + kind := "unknown" + + for _, c := range node.Status.Conditions { + if c.Reason == "CiliumIsUp" { + kind = "cilium" + break + } + if c.Reason == "CalicoIsUp" { + kind = "calico" + break + } + if c.Reason == "FlannelIsUp" { + kind = "flannel" + break + } + } + return kind +} + +// Convert an IPV4 string to a fake MAC address. +func ipv4ToMac(addr string) string { + ip := strings.Split(addr, ".") + if len(ip) != 4 { + return "" + } + var intIP [4]int + for i, val := range ip { + intIP[i], _ = strconv.Atoi(val) + } + return fmt.Sprintf("0a:0a:%02x:%02x:%02x:%02x", intIP[0], intIP[1], intIP[2], intIP[3]) +} diff --git a/internal/pkg/utils.go b/internal/pkg/utils.go index d39446c..305546b 100644 --- a/internal/pkg/utils.go +++ b/internal/pkg/utils.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" + f5_bigip "github.com/f5devcentral/f5-bigip-rest-go/bigip" "github.com/f5devcentral/f5-bigip-rest-go/deployer" "github.com/f5devcentral/f5-bigip-rest-go/utils" v1 "k8s.io/api/core/v1" @@ -217,6 +218,15 @@ func DeployForEvent(ctx context.Context, impactedClasses []string, apply func() } } + // TODO: fix the issue: + // 2023/06/19 09:26:49.824036 [ERROR] [cd7c411d-e392-424f-8a76-be055f1286d2] \ + // failed to deploy partition cis-c-tenant: 400, {"code":400,"message":"0107082a:3: \ + // All objects must be removed from a partition (cis-c-tenant) before the partition may be removed, type ID (973)","errorStack":[],"apiError":3} + + // TODO: fix the issue: + // 2023/06/19 09:17:39.572853 [ERROR] [a763bd16-498a-415a-89a3-f5fdf2aa5adf] \ + // failed to do deployment to https://10.250.15.109:443: 400, {"code":400,"message":"transaction failed:01070110:3: \ + // Node address '/cis-c-tenant/10.250.16.103' is referenced by a member of pool '/cis-c-tenant/default.dev-service'.","errorStack":[],"apiError":2} drs["cis-c-tenant"] = &deployer.DeployRequest{ Meta: fmt.Sprintf("Updating pools for event %s", meta), From: &opcfgs, @@ -226,7 +236,7 @@ func DeployForEvent(ctx context.Context, impactedClasses []string, apply func() } for _, dr := range drs { - PendingDeploys <- *dr + PendingDeploys.Add(*dr) } return nil @@ -291,3 +301,67 @@ func validateSecretType(group *gatewayv1beta1.Group, kind *gatewayv1beta1.Kind) } return nil } + +// purgeCommonNodes tries to remove nodes from Common if no reference. +func purgeCommonNodes(ctx context.Context, ombs []interface{}) { + for _, bp := range BIGIPs { + bc := f5_bigip.BIGIPContext{Context: ctx, BIGIP: *bp} + slog := utils.LogFromContext(ctx) + + for _, m := range ombs { + partition := m.(map[string]interface{})["partition"].(string) + if partition != "Common" { + continue + } + addr := m.(map[string]interface{})["address"].(string) + err := bc.Delete("ltm/node", addr, "Common", "") + if err != nil && !strings.Contains(err.Error(), "is referenced by a member of pool") { + slog.Warnf("cannot delete node %s: %s", addr, err.Error()) + } + } + } +} + +// // splitByPartition split the cfgs into a map of which keys are partitions +// func splitByPartition(ctx context.Context, cfgs map[string]interface{}) map[string]interface{} { +// partitions := map[string]map[string]map[string]interface{}{} +// for fstr, fv := range cfgs { +// for rstr, rv := range fv.(map[string]interface{}) { +// pstr := "unknown" +// if p, f := rv.(map[string]interface{})["partition"]; f { +// pstr = p.(string) +// } + +// if _, pok := partitions[pstr]; !pok { +// partitions[pstr] = map[string]map[string]interface{}{} + +// } +// if _, fok := partitions[pstr][fstr]; !fok { +// partitions[pstr][fstr] = map[string]interface{}{} +// } +// partitions[pstr][fstr][rstr] = rv +// } +// } +// rlt := map[string]interface{}{} +// for p, v := range partitions { +// rlt[p] = v +// } +// return rlt +// } + +// filterCommonResources filter the 'Common' resources from cfgs +func filterCommonResources(cfgs map[string]interface{}) map[string]interface{} { + rlt := map[string]interface{}{} + for fstr, fv := range cfgs { + if _, ok := rlt[fstr]; !ok { + rlt[fstr] = map[string]interface{}{} + } + for rstr, rv := range fv.(map[string]interface{}) { + if p, f := rv.(map[string]interface{})["partition"]; f && p == "Common" { + rlt[fstr].(map[string]interface{})[rstr] = rv + delete(cfgs[fstr].(map[string]interface{}), rstr) + } + } + } + return rlt +} diff --git a/internal/pkg/utils_test.go b/internal/pkg/utils_test.go index 05b411f..991a2db 100644 --- a/internal/pkg/utils_test.go +++ b/internal/pkg/utils_test.go @@ -101,3 +101,78 @@ func TestReferenceGrantFromTo_ops(t *testing.T) { } }) } + +func Test_filterCommonResources(t *testing.T) { + type args struct { + cfgs map[string]interface{} + } + tests := []struct { + name string + args args + want map[string]interface{} + left map[string]interface{} + }{ + { + name: "normal test", + args: args{ + cfgs: map[string]interface{}{ + "": map[string]interface{}{ + "ltm/pool/a": map[string]interface{}{ + "name": "a", + "partition": "cis-c-tenant", + }, + "ltm/pool/b": map[string]interface{}{ + "name": "b", + "partition": "Common", + }, + }, + "f": map[string]interface{}{ + "ltm/pool/c": map[string]interface{}{ + "name": "c", + "partition": "cis-c-tenant", + }, + "ltm/pool/d": map[string]interface{}{ + "name": "d", + "partition": "Common", + }, + }, + }, + }, + want: map[string]interface{}{ + "": map[string]interface{}{ + "ltm/pool/b": map[string]interface{}{ + "name": "b", + "partition": "Common", + }, + }, + "f": map[string]interface{}{ + "ltm/pool/d": map[string]interface{}{ + "name": "d", + "partition": "Common", + }, + }, + }, + left: map[string]interface{}{ + "": map[string]interface{}{ + "ltm/pool/a": map[string]interface{}{ + "name": "a", + "partition": "cis-c-tenant", + }, + }, + "f": map[string]interface{}{ + "ltm/pool/c": map[string]interface{}{ + "name": "c", + "partition": "cis-c-tenant", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := filterCommonResources(tt.args.cfgs); !reflect.DeepEqual(got, tt.want) || !reflect.DeepEqual(tt.args.cfgs, tt.left) { + t.Errorf("filterCommonResources() = %v, want %v, left %v", got, tt.want, tt.left) + } + }) + } +} diff --git a/internal/pkg/vars.go b/internal/pkg/vars.go index 0ddf4f6..2f65d47 100644 --- a/internal/pkg/vars.go +++ b/internal/pkg/vars.go @@ -2,12 +2,12 @@ package pkg import ( f5_bigip "github.com/f5devcentral/f5-bigip-rest-go/bigip" - "github.com/f5devcentral/f5-bigip-rest-go/deployer" + "github.com/f5devcentral/f5-bigip-rest-go/utils" ) var ( - PendingDeploys chan deployer.DeployRequest - DoneDeploys *deployer.DeployResponses + PendingDeploys *utils.DeployQueue + DoneDeploys *utils.DeployQueue ActiveSIGs *SIGCache BIGIPs []*f5_bigip.BIGIP BIPConfigs BIGIPConfigs