From 3dabbdb669542f6bd3bbda549fe196e06825e627 Mon Sep 17 00:00:00 2001 From: Chun Chen Date: Mon, 19 Oct 2020 19:21:26 +0800 Subject: [PATCH 1/3] Support allocate multiple ips --- pkg/api/galaxy/constant/constant.go | 79 +++--- pkg/api/galaxy/constant/constant_test.go | 51 ---- pkg/galaxy/server.go | 10 +- pkg/ipam/api/pool.go | 2 +- pkg/ipam/floatingip/ipam.go | 10 +- pkg/ipam/floatingip/ipam_crd.go | 250 ++++++++++++++---- pkg/ipam/schedulerplugin/event.go | 1 + pkg/ipam/schedulerplugin/floatingip_plugin.go | 123 +++++---- .../schedulerplugin/floatingip_plugin_test.go | 19 +- pkg/ipam/schedulerplugin/ipam.go | 30 ++- pkg/ipam/schedulerplugin/resync.go | 15 +- 11 files changed, 370 insertions(+), 220 deletions(-) delete mode 100644 pkg/api/galaxy/constant/constant_test.go diff --git a/pkg/api/galaxy/constant/constant.go b/pkg/api/galaxy/constant/constant.go index 613df1fc..697f406b 100644 --- a/pkg/api/galaxy/constant/constant.go +++ b/pkg/api/galaxy/constant/constant.go @@ -34,18 +34,7 @@ const ( // For fip crd object which has this label, it's reserved by admin manually. IPAM will not allocate it to pods. ReserveFIPLabel = "reserved" -) - -// ParseExtendedCNIArgs parses extended cni args from pod annotation -func ParseExtendedCNIArgs(args string) (map[string]map[string]json.RawMessage, error) { - argsMap := map[string]map[string]json.RawMessage{} - if err := json.Unmarshal([]byte(args), &argsMap); err != nil { - return nil, fmt.Errorf("failed to unmarshal %s value %s: %v", ExtendedCNIArgsAnnotation, args, err) - } - return argsMap, nil -} -const ( IPInfosKey = "ipinfos" ) @@ -56,38 +45,6 @@ type IPInfo struct { Gateway net.IP `json:"gateway"` } -// FormatIPInfo formats ipInfos as extended CNI Args annotation value -func FormatIPInfo(ipInfos []IPInfo) (string, error) { - data, err := json.Marshal(ipInfos) - if err != nil { - return "", err - } - raw := json.RawMessage(data) - str, err := json.Marshal(map[string]map[string]*json.RawMessage{CommonCNIArgsKey: {IPInfosKey: &raw}}) - return string(str), err -} - -// ParseIPInfo pareses ipInfo from annotation -func ParseIPInfo(str string) ([]IPInfo, error) { - m := map[string]map[string]*json.RawMessage{} - if err := json.Unmarshal([]byte(str), &m); err != nil { - return nil, fmt.Errorf("failed to unmarshal %s value %s: %v", ExtendedCNIArgsAnnotation, str, err) - } - commonMap := m[CommonCNIArgsKey] - if commonMap == nil { - return []IPInfo{}, nil - } - ipInfoStr := commonMap[IPInfosKey] - if ipInfoStr == nil { - return []IPInfo{}, nil - } - var ipInfos []IPInfo - if err := json.Unmarshal([]byte(*ipInfoStr), &ipInfos); err != nil { - return nil, fmt.Errorf("failed to unmarshal %s value %s as common/ipInfos: %v", ExtendedCNIArgsAnnotation, str, err) - } - return ipInfos, nil -} - // ReleasePolicy defines floatingip release policy type ReleasePolicy uint16 @@ -124,3 +81,39 @@ const ( NameSpace = "floating-ip" IpType = "ipType" ) + +// CniArgs is the cni args in pod annotation +type CniArgs struct { + // RequestIPRange is the requested ip candidates to allocate, one ip per []nets.IPRange + RequestIPRange [][]nets.IPRange `json:"request_ip_range,omitempty"` + // Common is the common args for cni plugins to setup network + Common CommonCniArgs `json:"common"` +} + +type CommonCniArgs struct { + IPInfos []IPInfo `json:"ipinfos,omitempty"` +} + +// UnmarshalCniArgs unmarshal cni args from input str +func UnmarshalCniArgs(str string) (*CniArgs, error) { + if str == "" { + return nil, nil + } + var cniArgs CniArgs + if err := json.Unmarshal([]byte(str), &cniArgs); err != nil { + return nil, fmt.Errorf("unmarshal pod cni args: %v", err) + } + return &cniArgs, nil +} + +// MarshalCniArgs marshal cni args of the given ipInfos +func MarshalCniArgs(ipInfos []IPInfo) (string, error) { + cniArgs := CniArgs{Common: CommonCniArgs{ + IPInfos: ipInfos, + }} + data, err := json.Marshal(cniArgs) + if err != nil { + return "", err + } + return string(data), nil +} diff --git a/pkg/api/galaxy/constant/constant_test.go b/pkg/api/galaxy/constant/constant_test.go deleted file mode 100644 index 2ea59d48..00000000 --- a/pkg/api/galaxy/constant/constant_test.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making TKEStack available. - * - * Copyright (C) 2012-2019 Tencent. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use - * this file except in compliance with the License. You may obtain a copy of the - * License at - * - * https://opensource.org/licenses/Apache-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package constant - -import ( - "net" - "reflect" - "testing" - - "tkestack.io/galaxy/pkg/utils/nets" -) - -func TestFormatParseIPInfo(t *testing.T) { - testCase := []IPInfo{ - { - IP: nets.NetsIPNet(&net.IPNet{IP: net.ParseIP("192.168.0.2"), Mask: net.IPv4Mask(255, 255, 0, 0)}), - Vlan: 2, - Gateway: net.ParseIP("192.168.0.1"), - }, - { - IP: nets.NetsIPNet(&net.IPNet{IP: net.ParseIP("192.168.0.3"), Mask: net.IPv4Mask(255, 255, 0, 0)}), - Vlan: 3, - Gateway: net.ParseIP("192.168.0.1"), - }, - } - str, err := FormatIPInfo(testCase) - if err != nil { - t.Fatal(err) - } - parsed, err := ParseIPInfo(str) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(parsed, testCase) { - t.Fatalf("real: %v, expect: %v", parsed, testCase) - } -} diff --git a/pkg/galaxy/server.go b/pkg/galaxy/server.go index 88c51d84..c57512a6 100644 --- a/pkg/galaxy/server.go +++ b/pkg/galaxy/server.go @@ -267,13 +267,13 @@ func parseExtendedCNIArgs(pod *corev1.Pod) (map[string]map[string]json.RawMessag if pod.Annotations == nil { return nil, nil } - annotation := pod.Annotations[constant.ExtendedCNIArgsAnnotation] - if annotation == "" { + args := pod.Annotations[constant.ExtendedCNIArgsAnnotation] + if args == "" { return nil, nil } - argsMap, err := constant.ParseExtendedCNIArgs(annotation) - if err != nil { - return nil, err + argsMap := map[string]map[string]json.RawMessage{} + if err := json.Unmarshal([]byte(args), &argsMap); err != nil { + return nil, fmt.Errorf("failed to unmarshal cni args %s: %v", args, err) } return argsMap, nil } diff --git a/pkg/ipam/api/pool.go b/pkg/ipam/api/pool.go index c16fe925..a02693fb 100644 --- a/pkg/ipam/api/pool.go +++ b/pkg/ipam/api/pool.go @@ -144,7 +144,7 @@ func (c *PoolController) preAllocateIP(req *restful.Request, resp *restful.Respo httputil.InternalError(resp, err) return } - subnetSet, err := c.IPAM.NodeSubnetsByKey("") + subnetSet, err := c.IPAM.NodeSubnetsByKeyAndIPRanges("", nil) if err != nil { httputil.InternalError(resp, err) return diff --git a/pkg/ipam/floatingip/ipam.go b/pkg/ipam/floatingip/ipam.go index 7ae11368..504bd9d3 100644 --- a/pkg/ipam/floatingip/ipam.go +++ b/pkg/ipam/floatingip/ipam.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/sets" "tkestack.io/galaxy/pkg/api/galaxy/constant" + "tkestack.io/galaxy/pkg/utils/nets" ) var ( @@ -42,6 +43,8 @@ type IPAM interface { AllocateSpecificIP(string, net.IP, Attr) error // AllocateInSubnet allocate subnet of IPs. AllocateInSubnet(string, *net.IPNet, Attr) (net.IP, error) + // AllocateInSubnetsAndIPRange allocates an ip for each ip range array of the input node subnet. + AllocateInSubnetsAndIPRange(string, *net.IPNet, [][]nets.IPRange, Attr) ([]net.IP, error) // AllocateInSubnetWithKey allocate a floatingIP in given subnet and key. AllocateInSubnetWithKey(oldK, newK, subnet string, attr Attr) error // ReserveIP can reserve a IP entitled by a terminated pod. Attributes **expect policy attr** will be updated. @@ -59,10 +62,13 @@ type IPAM interface { ByPrefix(string) ([]FloatingIP, error) // ByKeyword returns floatingIP set by a given keyword. ByKeyword(string) ([]FloatingIP, error) + // ByKeyAndIPRanges finds an ip for each iprange array by key, and returns all fips + ByKeyAndIPRanges(string, [][]nets.IPRange) ([]*FloatingIPInfo, error) // NodeSubnets returns node's subnet. NodeSubnet(net.IP) *net.IPNet - // NodeSubnetsByKey returns keys corresponding node subnets which has `key` as a prefix. - NodeSubnetsByKey(key string) (sets.String, error) + // NodeSubnetsByKeyAndIPRanges finds an ip for each iprange array by key, and returns their intersection + // node subnets. + NodeSubnetsByKeyAndIPRanges(key string, ipranges [][]nets.IPRange) (sets.String, error) // Name returns IPAM's name. Name() string // implements metrics Collector interface diff --git a/pkg/ipam/floatingip/ipam_crd.go b/pkg/ipam/floatingip/ipam_crd.go index 3a988c9b..792693db 100644 --- a/pkg/ipam/floatingip/ipam_crd.go +++ b/pkg/ipam/floatingip/ipam_crd.go @@ -181,6 +181,8 @@ func (ci *crdIpam) AllocateInSubnetWithKey(oldK, newK, subnet string, attr Attr) func (ci *crdIpam) ReserveIP(oldK, newK string, attr Attr) (bool, error) { ci.cacheLock.Lock() defer ci.cacheLock.Unlock() + date := time.Now() + var reserved bool for k, v := range ci.allocatedFIPs { if v.Key == oldK { if oldK == newK && v.PodUid == attr.Uid && v.NodeName == attr.NodeName { @@ -188,15 +190,17 @@ func (ci *crdIpam) ReserveIP(oldK, newK string, attr Attr) (bool, error) { return false, nil } attr.Policy = constant.ReleasePolicy(v.Policy) - date := time.Now() if err := ci.updateFloatingIP(v.CloneWith(newK, &attr, date)); err != nil { glog.Errorf("failed to update floatingIP %s: %v", k, err) return false, err } v.Assign(newK, &attr, date) - return true, nil + reserved = true } } + if reserved { + return true, nil + } return false, fmt.Errorf("failed to find floatIP by key %s", oldK) } @@ -319,11 +323,52 @@ func (ci *crdIpam) NodeSubnet(nodeIP net.IP) *net.IPNet { return nil } -func (ci *crdIpam) NodeSubnetsByKey(key string) (sets.String, error) { - if key == "" { - return ci.filterUnallocatedSubnet(), nil +func (ci *crdIpam) NodeSubnetsByKeyAndIPRanges(key string, ipranges [][]nets.IPRange) (sets.String, error) { + subnetSet := sets.NewString() + ci.cacheLock.RLock() + defer ci.cacheLock.RUnlock() + if len(ipranges) == 0 { + if key == "" { + for _, val := range ci.unallocatedFIPs { + subnetSet.Insert(val.Subnets.UnsortedList()...) + } + } else { + // key is not be empty + for _, spec := range ci.allocatedFIPs { + if spec.Key == key { + subnetSet.Insert(spec.Subnets.UnsortedList()...) + } + } + } + return subnetSet, nil + } + for i, ranges := range ipranges { + partSubnets := sets.NewString() + walkIPRanges(ranges, func(ip net.IP) bool { + ipStr := ip.String() + if key == "" { + if fip, ok := ci.unallocatedFIPs[ipStr]; !ok { + return false + } else { + partSubnets.Insert(fip.Subnets.UnsortedList()...) + } + } else { + // TODO if there is an allocated ip in iprange1 and no allocated ip in iprange2 + if fip, ok := ci.allocatedFIPs[ipStr]; !ok || fip.Key != key { + return false + } else { + partSubnets.Insert(fip.Subnets.UnsortedList()...) + } + } + return false + }) + if i == 0 { + subnetSet = partSubnets + } else { + subnetSet = subnetSet.Intersection(partSubnets) + } } - return ci.filterAllocatedSubnet(key), nil + return subnetSet, nil } // Shutdown shutdowns IPAM. @@ -412,24 +457,20 @@ func (ci *crdIpam) ConfigurePool(floatIPs []*FloatingIPPool) error { tmpCacheUnallocated := make(map[string]*FloatingIP) for i, fipConf := range floatIPs { subnetSet := nodeSubnets[i] - for _, ipr := range fipConf.IPRanges { - first := nets.IPToInt(ipr.First) - last := nets.IPToInt(ipr.Last) - for ; first <= last; first++ { - ip := nets.IntToIP(first) - ipStr := ip.String() - if _, contain := ci.allocatedFIPs[ipStr]; !contain { - tmpFip := &FloatingIP{ - IP: ip, - Key: "", - Policy: uint16(constant.ReleasePolicyPodDelete), - Subnets: subnetSet, - UpdatedAt: now, - } - tmpCacheUnallocated[ipStr] = tmpFip + walkIPRanges(fipConf.IPRanges, func(ip net.IP) bool { + ipStr := ip.String() + if _, contain := ci.allocatedFIPs[ipStr]; !contain { + tmpFip := &FloatingIP{ + IP: ip, + Key: "", + Policy: uint16(constant.ReleasePolicyPodDelete), + Subnets: subnetSet, + UpdatedAt: now, } + tmpCacheUnallocated[ipStr] = tmpFip } - } + return false + }) } ci.unallocatedFIPs = tmpCacheUnallocated return nil @@ -455,31 +496,6 @@ func (ci *crdIpam) syncCacheAfterDel(released *FloatingIP) { return } -func (ci *crdIpam) filterAllocatedSubnet(key string) sets.String { - //key would not be empty - subnetSet := sets.NewString() - ci.cacheLock.RLock() - defer ci.cacheLock.RUnlock() - for _, spec := range ci.allocatedFIPs { - if spec.Key == key { - subnetSet.Insert(spec.Subnets.UnsortedList()...) - } - } - return subnetSet -} - -// Sometimes unallocated subnet(key equals "") is needed, -// it will filter all subnet in unallocated floatingIP in cache -func (ci *crdIpam) filterUnallocatedSubnet() sets.String { - subnetSet := sets.NewString() - ci.cacheLock.RLock() - for _, val := range ci.unallocatedFIPs { - subnetSet.Insert(val.Subnets.UnsortedList()...) - } - ci.cacheLock.RUnlock() - return subnetSet -} - // ByKeyword returns floatingIP set by a given keyword. func (ci *crdIpam) ByKeyword(keyword string) ([]FloatingIP, error) { //not implement @@ -571,3 +587,143 @@ func (ci *crdIpam) Collect(ch chan<- prometheus.Metric) { "total", subnetStr, firstIP) } } + +// AllocateInSubnetsAndIPRange allocates an ip for each ip range array of the input node subnet. +func (ci *crdIpam) AllocateInSubnetsAndIPRange(key string, nodeSubnet *net.IPNet, ipranges [][]nets.IPRange, + attr Attr) ([]net.IP, error) { + if nodeSubnet == nil { + // this should never happen + return nil, fmt.Errorf("nil nodeSubnet") + } + if len(ipranges) == 0 { + ip, err := ci.AllocateInSubnet(key, nodeSubnet, attr) + if err != nil { + return nil, err + } + return []net.IP{ip}, nil + } + ci.cacheLock.Lock() + defer ci.cacheLock.Unlock() + // pick ips to allocate, one per []nets.IPRange + var allocatedIPStrs []string + for _, ranges := range ipranges { + var allocated bool + walkIPRanges(ranges, func(ip net.IP) bool { + ipStr := ip.String() + if fip, ok := ci.unallocatedFIPs[ipStr]; !ok || !fip.Subnets.Has(nodeSubnet.String()) { + return false + } + allocatedIPStrs = append(allocatedIPStrs, ipStr) + allocated = true + return true + }) + if !allocated { + glog.Warningf("not enough ip to allocate for %s in %s, %v", key, nodeSubnet.String(), ipranges) + return nil, ErrNoEnoughIP + } + } + var allocatedIPs []net.IP + var allocatedFips []*FloatingIP + // allocate all ips in crd before sync cache in memory + for i, allocatedIPStr := range allocatedIPStrs { + v := ci.unallocatedFIPs[allocatedIPStr] + // we never updates ip or subnet object, it's ok to share these objs. + allocated := New(v.IP, v.Subnets, key, &attr, time.Now()) + if err := ci.createFloatingIP(allocated); err != nil { + glog.Errorf("failed to create floatingIP %s: %v", allocatedIPStr, err) + // rollback all allocated ips + for j := range allocatedIPStrs { + if j == i { + break + } + if err := ci.deleteFloatingIP(allocatedIPStrs[j]); err != nil { + glog.Errorf("failed to delete floatingIP %s: %v", allocatedIPStrs[j], err) + } + } + return nil, err + } + allocatedIPs = append(allocatedIPs, v.IP) + allocatedFips = append(allocatedFips, allocated) + } + // sync cache when crds created + for i := range allocatedFips { + ci.syncCacheAfterCreate(allocatedFips[i]) + } + return allocatedIPs, nil +} + +// ByKeyAndIPRanges finds an ip for each iprange array by key, and returns all fips +func (ci *crdIpam) ByKeyAndIPRanges(key string, ipranges [][]nets.IPRange) ([]*FloatingIPInfo, error) { + ci.cacheLock.RLock() + defer ci.cacheLock.RUnlock() + var ipinfos []*FloatingIPInfo + if len(ipranges) != 0 { + for _, ranges := range ipranges { + var err error + walkIPRanges(ranges, func(ip net.IP) bool { + ipStr := ip.String() + fip := ci.allocatedFIPs[ipStr] + if fip.Key != key { + return false + } + fipInfo := ci.toFloatingIPInfo(fip) + if fipInfo == nil { + err = fmt.Errorf("could not find match floating ip config for ip %s", fip.IP.String()) + } else { + ipinfos = append(ipinfos, fipInfo) + } + return true + }) + if err != nil { + return nil, err + } + } + } else { + for _, fip := range ci.allocatedFIPs { + if fip.Key != key { + continue + } + fipInfo := ci.toFloatingIPInfo(fip) + if fipInfo == nil { + return nil, fmt.Errorf("could not find match floating ip config for ip %s", fip.IP.String()) + } else { + ipinfos = append(ipinfos, fipInfo) + } + } + } + return ipinfos, nil +} + +func (ci *crdIpam) toFloatingIPInfo(fip *FloatingIP) *FloatingIPInfo { + for _, fipPool := range ci.FloatingIPs { + if fipPool.Contains(fip.IP) { + ip := nets.IPNet(net.IPNet{ + IP: fip.IP, + Mask: fipPool.Mask, + }) + return &FloatingIPInfo{ + IPInfo: constant.IPInfo{ + IP: &ip, + Vlan: fipPool.Vlan, + Gateway: fipPool.Gateway, + }, + FIP: *fip, + } + } + } + return nil +} + +// walkIPRanges walks all ips in the ranges, and calls f for each ip. If f returns true, walkIPRanges stops. +func walkIPRanges(ranges []nets.IPRange, f func(ip net.IP) bool) { + for _, r := range ranges { + first := nets.IPToInt(r.First) + last := nets.IPToInt(r.Last) + for ; first <= last; first++ { + ip := nets.IntToIP(first) + if f(ip) { + return + } + } + } +} diff --git a/pkg/ipam/schedulerplugin/event.go b/pkg/ipam/schedulerplugin/event.go index d800a9fe..c52fc63e 100644 --- a/pkg/ipam/schedulerplugin/event.go +++ b/pkg/ipam/schedulerplugin/event.go @@ -45,6 +45,7 @@ func (p *FloatingIPPlugin) UpdatePod(oldPod, newPod *corev1.Pod) error { // If it's a evicted one, release its ip glog.Infof("release ip from %s_%s, phase %s", newPod.Name, newPod.Namespace, string(newPod.Status.Phase)) p.unreleased <- &releaseEvent{pod: newPod} + return nil } if err := p.syncPodIP(newPod); err != nil { glog.Warningf("failed to sync pod ip: %v", err) diff --git a/pkg/ipam/schedulerplugin/floatingip_plugin.go b/pkg/ipam/schedulerplugin/floatingip_plugin.go index 2e3e525d..ee8d76d5 100644 --- a/pkg/ipam/schedulerplugin/floatingip_plugin.go +++ b/pkg/ipam/schedulerplugin/floatingip_plugin.go @@ -17,6 +17,7 @@ package schedulerplugin import ( + "encoding/json" "fmt" "net" "sync" @@ -189,8 +190,12 @@ func (p *FloatingIPPlugin) getSubnet(pod *corev1.Pod) (sets.String, error) { if err != nil { return nil, err } + cniArgs, err := getPodCniArgs(pod) + if err != nil { + return nil, err + } // first check if exists an already allocated ip for this pod - subnets, err := p.ipam.NodeSubnetsByKey(keyObj.KeyInDB) + subnets, err := p.ipam.NodeSubnetsByKeyAndIPRanges(keyObj.KeyInDB, cniArgs.RequestIPRange) if err != nil { return nil, fmt.Errorf("failed to query by key %s: %v", keyObj.KeyInDB, err) } @@ -212,7 +217,7 @@ func (p *FloatingIPPlugin) getSubnet(pod *corev1.Pod) (sets.String, error) { // Lock to make checking available subnets and allocating reserved ip atomic defer p.LockDpPool(keyObj.PoolPrefix())() } - subnetSet, reserve, err := p.getAvailableSubnet(keyObj, policy, replicas, isPoolSizeDefined) + subnetSet, reserve, err := p.getAvailableSubnet(keyObj, policy, replicas, isPoolSizeDefined, cniArgs.RequestIPRange) if err != nil { return nil, err } @@ -262,56 +267,70 @@ func (p *FloatingIPPlugin) Prioritize(pod *corev1.Pod, nodes []corev1.Node) (*sc return list, nil } -func (p *FloatingIPPlugin) allocateIP(key string, nodeName string, pod *corev1.Pod) (*constant.IPInfo, error) { +func (p *FloatingIPPlugin) allocateIP(key string, nodeName string, pod *corev1.Pod) (*constant.CniArgs, error) { var how string - ipInfo, err := p.ipam.First(key) + cniArgs, err := getPodCniArgs(pod) + if err != nil { + return nil, err + } + ipInfos, err := p.ipam.ByKeyAndIPRanges(key, cniArgs.RequestIPRange) if err != nil { return nil, fmt.Errorf("failed to query floating ip by key %s: %v", key, err) } started := time.Now() policy := parseReleasePolicy(&pod.ObjectMeta) attr := floatingip.Attr{Policy: policy, NodeName: nodeName, Uid: string(pod.UID)} - if ipInfo != nil { + if len(ipInfos) != 0 { how = "reused" - // check if uid missmatch, if we delete a statfulset/tapp and creates a same name statfulset/tapp immediately, - // galaxy-ipam may receive bind event for new pod early than deleting event for old pod - if ipInfo.FIP.PodUid != "" && ipInfo.FIP.PodUid != string(pod.GetUID()) { - return nil, fmt.Errorf("waiting for delete event of %s before reuse this ip", key) + for _, ipInfo := range ipInfos { + // check if uid missmatch, if we delete a statfulset/tapp and creates a same name statfulset/tapp immediately, + // galaxy-ipam may receive bind event for new pod early than deleting event for old pod + if ipInfo.FIP.PodUid != "" && ipInfo.FIP.PodUid != string(pod.GetUID()) { + return nil, fmt.Errorf("waiting for delete event of %s before reuse this ip", key) + } } } else { subnet, err := p.queryNodeSubnet(nodeName) if err != nil { return nil, err } - if err := p.allocateInSubnet(key, subnet, attr, "bind"); err != nil { + if _, err := p.ipam.AllocateInSubnetsAndIPRange(key, subnet, cniArgs.RequestIPRange, attr); err != nil { return nil, err } how = "allocated" - ipInfo, err = p.ipam.First(key) + ipInfos, err = p.ipam.ByKeyAndIPRanges(key, cniArgs.RequestIPRange) if err != nil { return nil, fmt.Errorf("failed to query floating ip by key %s: %v", key, err) } - if ipInfo == nil { + if len(ipInfos) == 0 { return nil, fmt.Errorf("nil floating ip for key %s: %v", key, err) } } - glog.Infof("AssignIP nodeName %s, ip %s, key %s", nodeName, ipInfo.IPInfo.IP.IP.String(), key) - if err := p.cloudProviderAssignIP(&rpc.AssignIPRequest{ - NodeName: nodeName, - IPAddress: ipInfo.IPInfo.IP.IP.String(), - }); err != nil { - // do not rollback allocated ip - return nil, fmt.Errorf("failed to assign ip %s to %s: %v", ipInfo.IPInfo.IP.IP.String(), key, err) - } - if how == "reused" { - glog.Infof("pod %s reused %s, updating attr to %v", key, ipInfo.IPInfo.IP.String(), attr) - if err := p.ipam.UpdateAttr(key, ipInfo.IPInfo.IP.IP, attr); err != nil { - return nil, fmt.Errorf("failed to update floating ip release policy: %v", err) + for _, ipInfo := range ipInfos { + glog.Infof("AssignIP nodeName %s, ip %s, key %s", nodeName, ipInfo.IPInfo.IP.IP.String(), key) + if err := p.cloudProviderAssignIP(&rpc.AssignIPRequest{ + NodeName: nodeName, + IPAddress: ipInfo.IPInfo.IP.IP.String(), + }); err != nil { + // do not rollback allocated ip + return nil, fmt.Errorf("failed to assign ip %s to %s: %v", ipInfo.IPInfo.IP.IP.String(), key, err) + } + if how == "reused" { + glog.Infof("pod %s reused %s, updating attr to %v", key, ipInfo.IPInfo.IP.String(), attr) + if err := p.ipam.UpdateAttr(key, ipInfo.IPInfo.IP.IP, attr); err != nil { + return nil, fmt.Errorf("failed to update floating ip release policy: %v", err) + } } } - glog.Infof("started at %d %s ip %s, attr %v for %s", started.UnixNano(), how, - ipInfo.IPInfo.IP.String(), attr, key) - return &ipInfo.IPInfo, nil + var ips []string + var ret []constant.IPInfo + for _, ipInfo := range ipInfos { + ips = append(ips, ipInfo.IPInfo.IP.String()) + ret = append(ret, ipInfo.IPInfo) + } + glog.Infof("started at %d %s ips %s, attr %v for %s", started.UnixNano(), how, ips, attr, key) + cniArgs.Common.IPInfos = ret + return &cniArgs, nil } // Bind binds a new floatingip or reuse an old one to pod @@ -331,17 +350,15 @@ func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error { if err != nil { return err } - ipInfo, err := p.allocateIP(keyObj.KeyInDB, args.Node, pod) + cniArgs, err := p.allocateIP(keyObj.KeyInDB, args.Node, pod) if err != nil { return err } - ipInfos := []constant.IPInfo{*ipInfo} - bindAnnotation := make(map[string]string) - data, err := constant.FormatIPInfo(ipInfos) + data, err := json.Marshal(cniArgs) if err != nil { - return fmt.Errorf("failed to format ipinfo %v: %v", ipInfos, err) + return fmt.Errorf("marshal cni args %v: %v", *cniArgs, err) } - bindAnnotation[constant.ExtendedCNIArgsAnnotation] = data //TODO don't overlap this annotation + bindAnnotation := map[string]string{constant.ExtendedCNIArgsAnnotation: string(data)} var err1 error if err := wait.PollImmediate(time.Millisecond*500, 3*time.Second, func() (bool, error) { // It's the extender's response to bind pods to nodes since it is a binder @@ -390,21 +407,19 @@ func (p *FloatingIPPlugin) unbind(pod *corev1.Pod) error { } key := keyObj.KeyInDB if p.cloudProvider != nil { - ipInfo, err := p.ipam.First(key) + ipInfos, err := p.ipam.ByKeyAndIPRanges(key, nil) if err != nil { - return fmt.Errorf("failed to query floating ip of %s: %v", key, err) - } - if ipInfo == nil { - glog.Infof("pod %s hasn't an allocated ip", key) - return nil + return fmt.Errorf("query floating ip by key %s: %v", key, err) } - ipStr := ipInfo.IPInfo.IP.IP.String() - glog.Infof("UnAssignIP nodeName %s, ip %s, key %s", ipInfo.FIP.NodeName, ipStr, key) - if err = p.cloudProviderUnAssignIP(&rpc.UnAssignIPRequest{ - NodeName: ipInfo.FIP.NodeName, - IPAddress: ipStr, - }); err != nil { - return fmt.Errorf("failed to unassign ip %s from %s: %v", ipStr, key, err) + for _, ipInfo := range ipInfos { + ipStr := ipInfo.IPInfo.IP.IP.String() + glog.Infof("UnAssignIP nodeName %s, ip %s, key %s", ipInfo.FIP.NodeName, ipStr, key) + if err = p.cloudProviderUnAssignIP(&rpc.UnAssignIPRequest{ + NodeName: ipInfo.FIP.NodeName, + IPAddress: ipStr, + }); err != nil { + return fmt.Errorf("failed to unassign ip %s from %s: %v", ipStr, key, err) + } } } policy := parseReleasePolicy(&pod.ObjectMeta) @@ -490,3 +505,19 @@ func (p *FloatingIPPlugin) lockPod(name, namespace string) func() { _ = p.podLockPool.UnlockKey(key) } } + +func getPodCniArgs(pod *corev1.Pod) (constant.CniArgs, error) { + m := pod.GetAnnotations() + if len(m) == 0 { + return constant.CniArgs{}, nil + } + str, ok := m[constant.ExtendedCNIArgsAnnotation] + if !ok { + return constant.CniArgs{}, nil + } + args, err := constant.UnmarshalCniArgs(str) + if args == nil { + args = &constant.CniArgs{} + } + return *args, err +} diff --git a/pkg/ipam/schedulerplugin/floatingip_plugin_test.go b/pkg/ipam/schedulerplugin/floatingip_plugin_test.go index c15cf370..e65fe090 100644 --- a/pkg/ipam/schedulerplugin/floatingip_plugin_test.go +++ b/pkg/ipam/schedulerplugin/floatingip_plugin_test.go @@ -163,12 +163,12 @@ func TestAllocateIP(t *testing.T) { // check update from ReleasePolicyPodDelete to ReleasePolicyImmutable pod.Spec.NodeName = node4 pod.SetUID("pod-xx-1") - ipInfo, err := fipPlugin.allocateIP(podKey.KeyInDB, pod.Spec.NodeName, pod) - if err != nil { + cniArgs, err := fipPlugin.allocateIP(podKey.KeyInDB, pod.Spec.NodeName, pod) + if err != nil || len(cniArgs.Common.IPInfos) != 1 { t.Fatal(err) } - if ipInfo == nil || ipInfo.IP.String() != "10.173.13.2/24" { - t.Fatal(ipInfo) + if cniArgs.Common.IPInfos[0].IP.String() != "10.173.13.2/24" { + t.Fatal(cniArgs.Common.IPInfos[0]) } fip, err := fipPlugin.ipam.First(podKey.KeyInDB) if err != nil { @@ -194,7 +194,7 @@ func TestUpdatePod(t *testing.T) { if err := json.Unmarshal([]byte(`{"ip":"10.173.13.2/24","vlan":2,"gateway":"10.173.13.1","routable_subnet":"10.173.13.0/24"}`), &ipInfo); err != nil { t.Fatal() } - str, err := constant.FormatIPInfo([]constant.IPInfo{ipInfo}) + str, err := constant.MarshalCniArgs([]constant.IPInfo{ipInfo}) if err != nil { t.Fatal(err) } @@ -238,10 +238,11 @@ func TestFilterForDeployment(t *testing.T) { // pre-allocate ip in filter for deployment pod podKey, _ := schedulerplugin_util.FormatKey(pod) deadPodKey, _ := schedulerplugin_util.FormatKey(deadPod) - fip, err := fipPlugin.allocateIP(deadPodKey.KeyInDB, node3, deadPod) - if err != nil { + cniArgs, err := fipPlugin.allocateIP(deadPodKey.KeyInDB, node3, deadPod) + if err != nil || len(cniArgs.Common.IPInfos) != 1 { t.Fatal(err) } + fip := cniArgs.Common.IPInfos[0] // because deployment ip is allocated to deadPod, check if pod gets none available subnets filtered, failed, err := fipPlugin.Filter(pod, nodes) if err == nil || !strings.Contains(err.Error(), "wait for releasing") { @@ -552,7 +553,7 @@ func TestBind(t *testing.T) { t.Fatalf("Unexpected error: %v", err) return } - str, err := constant.FormatIPInfo([]constant.IPInfo{fipInfo.IPInfo}) + str, err := constant.MarshalCniArgs([]constant.IPInfo{fipInfo.IPInfo}) if err != nil { t.Fatal(err) } @@ -669,7 +670,7 @@ func TestUnBind(t *testing.T) { if err != nil { t.Fatal(err) } - str, err := constant.FormatIPInfo([]constant.IPInfo{fipInfo.IPInfo}) + str, err := constant.MarshalCniArgs([]constant.IPInfo{fipInfo.IPInfo}) if err != nil { t.Fatal(err) } diff --git a/pkg/ipam/schedulerplugin/ipam.go b/pkg/ipam/schedulerplugin/ipam.go index a61c1990..7dfbd718 100644 --- a/pkg/ipam/schedulerplugin/ipam.go +++ b/pkg/ipam/schedulerplugin/ipam.go @@ -29,6 +29,7 @@ import ( "tkestack.io/galaxy/pkg/api/galaxy/constant" "tkestack.io/galaxy/pkg/ipam/floatingip" "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" + "tkestack.io/galaxy/pkg/utils/nets" ) func (p *FloatingIPPlugin) ensureIPAMConf(lastConf *string, newConf string) (bool, error) { @@ -71,8 +72,13 @@ func (p *FloatingIPPlugin) allocateInSubnetWithKey(oldK, newK, subnet string, at // #lizard forgives func (p *FloatingIPPlugin) getAvailableSubnet(keyObj *util.KeyObj, policy constant.ReleasePolicy, replicas int, - isPoolSizeDefined bool) (subnets sets.String, reserve bool, err error) { + isPoolSizeDefined bool, ipranges [][]nets.IPRange) (subnets sets.String, reserve bool, err error) { if keyObj.Deployment() && policy != constant.ReleasePolicyPodDelete { + if len(ipranges) > 0 { + // this introduce lots of complexity, don't support it for now + return nil, false, fmt.Errorf("request ip ranges for deployment pod with release " + + "policy other than ReleasePolicyPodDelete is not supported") + } var ips []floatingip.FloatingIP poolPrefix := keyObj.PoolPrefix() poolAppPrefix := keyObj.PoolAppPrefix() @@ -111,7 +117,7 @@ func (p *FloatingIPPlugin) getAvailableSubnet(keyObj *util.KeyObj, policy consta return unusedSubnetSet, true, nil } } - if subnets, err = p.ipam.NodeSubnetsByKey(""); err != nil { + if subnets, err = p.ipam.NodeSubnetsByKeyAndIPRanges("", ipranges); err != nil { err = fmt.Errorf("failed to query allocatable subnet: %v", err) return } @@ -119,19 +125,21 @@ func (p *FloatingIPPlugin) getAvailableSubnet(keyObj *util.KeyObj, policy consta } func (p *FloatingIPPlugin) releaseIP(key string, reason string) error { - ipInfo, err := p.ipam.First(key) - if err != nil { - return fmt.Errorf("failed to query floating ip of %s: %v", key, err) - } - if ipInfo == nil { + ipInfos, err := p.ipam.ByKeyAndIPRanges(key, nil) + if len(ipInfos) == 0 { glog.Infof("release floating ip from %s because of %s, but already been released", key, reason) return nil } - if err := p.ipam.Release(key, ipInfo.IPInfo.IP.IP); err != nil { - return fmt.Errorf("failed to release floating ip of %s because of %s: %v", key, reason, err) + m := map[string]string{} + for i := range ipInfos { + m[ipInfos[i].FIP.IP.String()] = ipInfos[i].FIP.Key + } + released, unreleased, err := p.ipam.ReleaseIPs(m) + if err != nil { + return fmt.Errorf("released %v, unreleased %v of %s because of %s: %v", released, unreleased, key, + reason, err) } - glog.Infof("released floating ip %s from %s because of %s", ipInfo.IPInfo.IP.String(), key, - reason) + glog.Infof("released floating ip %v from %s because of %s", released, key, reason) return nil } diff --git a/pkg/ipam/schedulerplugin/resync.go b/pkg/ipam/schedulerplugin/resync.go index f58c8dc8..102422a7 100644 --- a/pkg/ipam/schedulerplugin/resync.go +++ b/pkg/ipam/schedulerplugin/resync.go @@ -197,15 +197,20 @@ func (p *FloatingIPPlugin) syncPodIP(pod *corev1.Pod) error { glog.V(5).Infof("sync pod %s/%s ip formatKey with error %v", pod.Namespace, pod.Name, err) return nil } - ipInfos, err := constant.ParseIPInfo(pod.Annotations[constant.ExtendedCNIArgsAnnotation]) + cniArgs, err := constant.UnmarshalCniArgs(pod.Annotations[constant.ExtendedCNIArgsAnnotation]) if err != nil { return err } - if len(ipInfos) == 0 || ipInfos[0].IP == nil { - // should not happen - return fmt.Errorf("empty ipinfo for pod %s", keyObj.KeyInDB) + ipInfos := cniArgs.Common.IPInfos + for i := range ipInfos { + if ipInfos[i].IP == nil || ipInfos[i].IP.IP == nil { + continue + } + if err := p.syncIP(keyObj.KeyInDB, ipInfos[i].IP.IP, pod); err != nil { + glog.Warningf("sync pod %s ip %s: %v", keyObj.KeyInDB, ipInfos[i].IP.IP.String(), err) + } } - return p.syncIP(keyObj.KeyInDB, ipInfos[0].IP.IP, pod) + return nil } func (p *FloatingIPPlugin) syncIP(key string, ip net.IP, pod *corev1.Pod) error { From ee242ab01ebb848e28fa78e34a469c203b6fe130 Mon Sep 17 00:00:00 2001 From: Chun Chen Date: Tue, 20 Oct 2020 12:04:35 +0800 Subject: [PATCH 2/3] Split floatingip_plugin.go and floatingip_plugin_test.go --- .../testing/fake_cloud_provider.go | 42 ++ pkg/ipam/schedulerplugin/bind.go | 193 ++++++ pkg/ipam/schedulerplugin/bind_test.go | 245 ++++++++ .../schedulerplugin/cloudprovider_test.go | 2 +- pkg/ipam/schedulerplugin/deployment_test.go | 2 +- pkg/ipam/schedulerplugin/filter.go | 144 +++++ pkg/ipam/schedulerplugin/filter_test.go | 279 +++++++++ pkg/ipam/schedulerplugin/floatingip_plugin.go | 279 --------- .../schedulerplugin/floatingip_plugin_test.go | 555 +----------------- pkg/ipam/schedulerplugin/testing/util.go | 52 ++ 10 files changed, 963 insertions(+), 830 deletions(-) create mode 100644 pkg/ipam/schedulerplugin/bind.go create mode 100644 pkg/ipam/schedulerplugin/bind_test.go create mode 100644 pkg/ipam/schedulerplugin/filter.go create mode 100644 pkg/ipam/schedulerplugin/filter_test.go diff --git a/pkg/ipam/cloudprovider/testing/fake_cloud_provider.go b/pkg/ipam/cloudprovider/testing/fake_cloud_provider.go index 2c705758..4affd759 100644 --- a/pkg/ipam/cloudprovider/testing/fake_cloud_provider.go +++ b/pkg/ipam/cloudprovider/testing/fake_cloud_provider.go @@ -15,3 +15,45 @@ * specific language governing permissions and limitations under the License. */ package testing + +import ( + "fmt" + + "tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc" +) + +// FakeCloudProvider is a fake cloud provider for testing +type FakeCloudProvider struct { + ExpectIP string + ExpectNode string + InvokedAssignIP bool + InvokedUnAssignIP bool +} + +func (f *FakeCloudProvider) AssignIP(in *rpc.AssignIPRequest) (*rpc.AssignIPReply, error) { + f.InvokedAssignIP = true + if in == nil { + return nil, fmt.Errorf("nil request") + } + if in.IPAddress != f.ExpectIP { + return nil, fmt.Errorf("expect ip %s, got %s", f.ExpectIP, in.IPAddress) + } + if in.NodeName != f.ExpectNode { + return nil, fmt.Errorf("expect node name %s, got %s", f.ExpectNode, in.NodeName) + } + return &rpc.AssignIPReply{Success: true}, nil +} + +func (f *FakeCloudProvider) UnAssignIP(in *rpc.UnAssignIPRequest) (*rpc.UnAssignIPReply, error) { + f.InvokedUnAssignIP = true + if in == nil { + return nil, fmt.Errorf("nil request") + } + if in.IPAddress != f.ExpectIP { + return nil, fmt.Errorf("expect ip %s, got %s", f.ExpectIP, in.IPAddress) + } + if in.NodeName != f.ExpectNode { + return nil, fmt.Errorf("expect node name %s, got %s", f.ExpectNode, in.NodeName) + } + return &rpc.UnAssignIPReply{Success: true}, nil +} diff --git a/pkg/ipam/schedulerplugin/bind.go b/pkg/ipam/schedulerplugin/bind.go new file mode 100644 index 00000000..b16a4ba2 --- /dev/null +++ b/pkg/ipam/schedulerplugin/bind.go @@ -0,0 +1,193 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package schedulerplugin + +import ( + "encoding/json" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + glog "k8s.io/klog" + "tkestack.io/galaxy/pkg/api/galaxy/constant" + "tkestack.io/galaxy/pkg/api/k8s/schedulerapi" + "tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc" + "tkestack.io/galaxy/pkg/ipam/floatingip" + "tkestack.io/galaxy/pkg/ipam/metrics" + "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" +) + +// Bind binds a new floatingip or reuse an old one to pod +func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error { + start := time.Now() + pod, err := p.PluginFactoryArgs.PodLister.Pods(args.PodNamespace).Get(args.PodName) + if err != nil { + return fmt.Errorf("failed to find pod %s: %w", util.Join(args.PodName, args.PodNamespace), err) + } + if !p.hasResourceName(&pod.Spec) { + // we will config extender resources which ensures pod which doesn't want floatingip won't be sent to plugin + // see https://github.com/kubernetes/kubernetes/pull/60332 + return fmt.Errorf("pod which doesn't want floatingip have been sent to plugin") + } + defer p.lockPod(pod.Name, pod.Namespace)() + keyObj, err := util.FormatKey(pod) + if err != nil { + return err + } + cniArgs, err := p.allocateIP(keyObj.KeyInDB, args.Node, pod) + if err != nil { + return err + } + data, err := json.Marshal(cniArgs) + if err != nil { + return fmt.Errorf("marshal cni args %v: %v", *cniArgs, err) + } + bindAnnotation := map[string]string{constant.ExtendedCNIArgsAnnotation: string(data)} + var err1 error + if err := wait.PollImmediate(time.Millisecond*500, 3*time.Second, func() (bool, error) { + // It's the extender's response to bind pods to nodes since it is a binder + if err := p.Client.CoreV1().Pods(args.PodNamespace).Bind(&corev1.Binding{ + ObjectMeta: v1.ObjectMeta{Namespace: args.PodNamespace, Name: args.PodName, UID: args.PodUID, + Annotations: bindAnnotation}, + Target: corev1.ObjectReference{ + Kind: "Node", + Name: args.Node, + }, + }); err != nil { + err1 = err + if apierrors.IsNotFound(err) { + // break retry if pod no longer exists + return false, err + } + return false, nil + } + glog.Infof("bind pod %s to %s with ip %v", keyObj.KeyInDB, args.Node, + bindAnnotation[constant.ExtendedCNIArgsAnnotation]) + return true, nil + }); err != nil { + if apierrors.IsNotFound(err1) { + glog.Infof("binding returns not found for pod %s, putting it into unreleased chan", keyObj.KeyInDB) + // attach ip annotation + p.unreleased <- &releaseEvent{pod: pod} + } + // If fails to update, depending on resync to update + return fmt.Errorf("update pod %s: %w", keyObj.KeyInDB, err1) + } + metrics.ScheduleLatency.WithLabelValues("bind").Observe(time.Since(start).Seconds()) + return nil +} + +func (p *FloatingIPPlugin) allocateIP(key string, nodeName string, pod *corev1.Pod) (*constant.CniArgs, error) { + var how string + cniArgs, err := getPodCniArgs(pod) + if err != nil { + return nil, err + } + ipInfos, err := p.ipam.ByKeyAndIPRanges(key, cniArgs.RequestIPRange) + if err != nil { + return nil, fmt.Errorf("failed to query floating ip by key %s: %v", key, err) + } + started := time.Now() + policy := parseReleasePolicy(&pod.ObjectMeta) + attr := floatingip.Attr{Policy: policy, NodeName: nodeName, Uid: string(pod.UID)} + if len(ipInfos) != 0 { + how = "reused" + for _, ipInfo := range ipInfos { + // check if uid missmatch, if we delete a statfulset/tapp and creates a same name statfulset/tapp immediately, + // galaxy-ipam may receive bind event for new pod early than deleting event for old pod + if ipInfo.FIP.PodUid != "" && ipInfo.FIP.PodUid != string(pod.GetUID()) { + return nil, fmt.Errorf("waiting for delete event of %s before reuse this ip", key) + } + } + } else { + subnet, err := p.queryNodeSubnet(nodeName) + if err != nil { + return nil, err + } + if _, err := p.ipam.AllocateInSubnetsAndIPRange(key, subnet, cniArgs.RequestIPRange, attr); err != nil { + return nil, err + } + how = "allocated" + ipInfos, err = p.ipam.ByKeyAndIPRanges(key, cniArgs.RequestIPRange) + if err != nil { + return nil, fmt.Errorf("failed to query floating ip by key %s: %v", key, err) + } + if len(ipInfos) == 0 { + return nil, fmt.Errorf("nil floating ip for key %s: %v", key, err) + } + } + for _, ipInfo := range ipInfos { + glog.Infof("AssignIP nodeName %s, ip %s, key %s", nodeName, ipInfo.IPInfo.IP.IP.String(), key) + if err := p.cloudProviderAssignIP(&rpc.AssignIPRequest{ + NodeName: nodeName, + IPAddress: ipInfo.IPInfo.IP.IP.String(), + }); err != nil { + // do not rollback allocated ip + return nil, fmt.Errorf("failed to assign ip %s to %s: %v", ipInfo.IPInfo.IP.IP.String(), key, err) + } + if how == "reused" { + glog.Infof("pod %s reused %s, updating attr to %v", key, ipInfo.IPInfo.IP.String(), attr) + if err := p.ipam.UpdateAttr(key, ipInfo.IPInfo.IP.IP, attr); err != nil { + return nil, fmt.Errorf("failed to update floating ip release policy: %v", err) + } + } + } + var ips []string + var ret []constant.IPInfo + for _, ipInfo := range ipInfos { + ips = append(ips, ipInfo.IPInfo.IP.String()) + ret = append(ret, ipInfo.IPInfo) + } + glog.Infof("started at %d %s ips %s, attr %v for %s", started.UnixNano(), how, ips, attr, key) + cniArgs.Common.IPInfos = ret + return &cniArgs, nil +} + +// unbind release ip from pod +func (p *FloatingIPPlugin) unbind(pod *corev1.Pod) error { + defer p.lockPod(pod.Name, pod.Namespace)() + glog.V(3).Infof("handle unbind pod %s", pod.Name) + keyObj, err := util.FormatKey(pod) + if err != nil { + return err + } + key := keyObj.KeyInDB + if p.cloudProvider != nil { + ipInfos, err := p.ipam.ByKeyAndIPRanges(key, nil) + if err != nil { + return fmt.Errorf("query floating ip by key %s: %v", key, err) + } + for _, ipInfo := range ipInfos { + ipStr := ipInfo.IPInfo.IP.IP.String() + glog.Infof("UnAssignIP nodeName %s, ip %s, key %s", ipInfo.FIP.NodeName, ipStr, key) + if err = p.cloudProviderUnAssignIP(&rpc.UnAssignIPRequest{ + NodeName: ipInfo.FIP.NodeName, + IPAddress: ipStr, + }); err != nil { + return fmt.Errorf("failed to unassign ip %s from %s: %v", ipStr, key, err) + } + } + } + policy := parseReleasePolicy(&pod.ObjectMeta) + if keyObj.Deployment() { + return p.unbindDpPod(keyObj, policy, "during unbinding pod") + } + return p.unbindNoneDpPod(keyObj, policy, "during unbinding pod") +} diff --git a/pkg/ipam/schedulerplugin/bind_test.go b/pkg/ipam/schedulerplugin/bind_test.go new file mode 100644 index 00000000..c7d51e14 --- /dev/null +++ b/pkg/ipam/schedulerplugin/bind_test.go @@ -0,0 +1,245 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package schedulerplugin + +import ( + "fmt" + "net" + "reflect" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + fakeV1 "k8s.io/client-go/kubernetes/typed/core/v1/fake" + "tkestack.io/galaxy/pkg/api/galaxy/constant" + "tkestack.io/galaxy/pkg/api/k8s/schedulerapi" + . "tkestack.io/galaxy/pkg/ipam/cloudprovider/testing" + "tkestack.io/galaxy/pkg/ipam/floatingip" + . "tkestack.io/galaxy/pkg/ipam/schedulerplugin/testing" + schedulerplugin_util "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" +) + +func TestBind(t *testing.T) { + fipPlugin, stopChan, _ := createPluginTestNodes(t, pod) + defer func() { stopChan <- struct{}{} }() + fipInfo, err := checkBind(fipPlugin, pod, node3, podKey.KeyInDB, node3Subnet) + if err != nil { + t.Fatalf("checkBind error %v", err) + } + fakePods := fipPlugin.PluginFactoryArgs.Client.CoreV1().Pods(pod.Namespace).(*fakeV1.FakePods) + + actualBinding, err := fakePods.GetBinding(pod.GetName()) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + return + } + str, err := constant.MarshalCniArgs([]constant.IPInfo{fipInfo.IPInfo}) + if err != nil { + t.Fatal(err) + } + expect := &corev1.Binding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: pod.Namespace, Name: pod.Name, + Annotations: map[string]string{ + constant.ExtendedCNIArgsAnnotation: str}}, + Target: corev1.ObjectReference{ + Kind: "Node", + Name: node3, + }, + } + if !reflect.DeepEqual(expect, actualBinding) { + t.Fatalf("Binding did not match expectation, expect %v, actual %v", expect, actualBinding) + } +} + +func TestAllocateIP(t *testing.T) { + fipPlugin, stopChan, _ := createPluginTestNodes(t) + defer func() { stopChan <- struct{}{} }() + + if err := fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), + floatingip.Attr{Policy: constant.ReleasePolicyPodDelete}); err != nil { + t.Fatal(err) + } + // check update from ReleasePolicyPodDelete to ReleasePolicyImmutable + pod.Spec.NodeName = node4 + pod.SetUID("pod-xx-1") + cniArgs, err := fipPlugin.allocateIP(podKey.KeyInDB, pod.Spec.NodeName, pod) + if err != nil || len(cniArgs.Common.IPInfos) != 1 { + t.Fatal(err) + } + if cniArgs.Common.IPInfos[0].IP.String() != "10.173.13.2/24" { + t.Fatal(cniArgs.Common.IPInfos[0]) + } + fip, err := fipPlugin.ipam.First(podKey.KeyInDB) + if err != nil { + t.Fatal(err) + } + if fip.FIP.Policy != uint16(constant.ReleasePolicyImmutable) { + t.Fatal(fip.FIP.Policy) + } + if fip.FIP.NodeName != node4 { + t.Fatal(fip.FIP.NodeName) + } + if fip.FIP.PodUid != string(pod.UID) { + t.Fatal(fip.FIP.PodUid) + } +} + +// #lizard forgives +func TestAllocateRecentIPs(t *testing.T) { + pod := CreateDeploymentPod("dp-xxx-yyy", "ns1", poolAnnotation("pool1")) + dp := CreateDeployment(pod.ObjectMeta, 1) + fipPlugin, stopChan, nodes := createPluginTestNodes(t, pod, dp) + defer func() { stopChan <- struct{}{} }() + podKey, _ := schedulerplugin_util.FormatKey(pod) + if err := fipPlugin.ipam.AllocateSpecificIP(podKey.PoolPrefix(), net.ParseIP("10.49.27.205"), + floatingip.Attr{Policy: constant.ReleasePolicyPodDelete}); err != nil { + t.Fatal(err) + } + // update time of 10.49.27.216 is more recently than 10.49.27.205 + if err := fipPlugin.ipam.AllocateSpecificIP(podKey.PoolPrefix(), net.ParseIP("10.49.27.216"), + floatingip.Attr{Policy: constant.ReleasePolicyPodDelete}); err != nil { + t.Fatal(err) + } + // check filter allocates recent ips for deployment pod from ip pool + if err := checkFilterCase(fipPlugin, filterCase{ + testPod: pod, expectFiltererd: []string{node3}, expectFailed: []string{drainedNode, nodeHasNoIP, node4}, + }, nodes); err != nil { + t.Fatal(err) + } + if err := checkIPKey(fipPlugin.ipam, "10.49.27.205", podKey.PoolPrefix()); err != nil { + t.Fatal(err) + } + if err := checkIPKey(fipPlugin.ipam, "10.49.27.216", podKey.KeyInDB); err != nil { + t.Fatal(err) + } +} + +// #lizard forgives +func TestUnBind(t *testing.T) { + pod1 := CreateStatefulSetPod("pod1-1", "demo", map[string]string{}) + keyObj, _ := schedulerplugin_util.FormatKey(pod1) + fipPlugin, stopChan, _ := createPluginTestNodes(t, pod1) + defer func() { stopChan <- struct{}{} }() + fipPlugin.cloudProvider = &FakeCloudProvider{} + // if a pod has not got cni args annotation, unbind should return nil + if err := fipPlugin.unbind(pod1); err != nil { + t.Fatal(err) + } + // if a pod has got bad cni args annotation, + // unbind should return nil because we got binded ip from store instead of annotation + pod1.Annotations[constant.ExtendedCNIArgsAnnotation] = "fff" + if err := fipPlugin.unbind(pod1); err != nil { + t.Fatal(err) + } + // bind before testing normal unbind + expectIP := net.ParseIP("10.49.27.205") + if err := drainNode(fipPlugin, node3Subnet, expectIP); err != nil { + t.Fatal(err) + } + fakeCP := &FakeCloudProvider{ExpectIP: expectIP.String(), ExpectNode: node3} + fipPlugin.cloudProvider = fakeCP + fipInfo, err := checkBind(fipPlugin, pod1, node3, keyObj.KeyInDB, node3Subnet) + if err != nil { + t.Fatal(err) + } + str, err := constant.MarshalCniArgs([]constant.IPInfo{fipInfo.IPInfo}) + if err != nil { + t.Fatal(err) + } + pod1.Annotations[constant.ExtendedCNIArgsAnnotation] = str + pod1.Spec.NodeName = node3 + if err := fipPlugin.unbind(pod1); err != nil { + t.Fatal(err) + } + if !fakeCP.InvokedAssignIP || !fakeCP.InvokedUnAssignIP { + t.Fatal() + } +} + +func TestUnBindImmutablePod(t *testing.T) { + pod = CreateStatefulSetPodWithLabels("sts1-0", "ns1", map[string]string{"app": "sts1"}, immutableAnnotation) + podKey, _ = schedulerplugin_util.FormatKey(pod) + fipPlugin, stopChan, _ := createPluginTestNodes(t, pod, CreateStatefulSet(pod.ObjectMeta, 1)) + defer func() { stopChan <- struct{}{} }() + if err := fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), + floatingip.Attr{Policy: constant.ReleasePolicyImmutable}); err != nil { + t.Fatal(err) + } + // unbind the pod, check ip should be reserved, because pod has is immutable + if err := fipPlugin.unbind(pod); err != nil { + t.Fatal(err) + } + if err := checkIPKey(fipPlugin.ipam, "10.173.13.2", podKey.KeyInDB); err != nil { + t.Fatal(err) + } +} + +func checkBind(fipPlugin *FloatingIPPlugin, pod *corev1.Pod, nodeName, checkKey string, + expectSubnet *net.IPNet) (*floatingip.FloatingIPInfo, error) { + if err := fipPlugin.Bind(&schedulerapi.ExtenderBindingArgs{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + Node: nodeName, + }); err != nil { + return nil, err + } + fipInfo, err := fipPlugin.ipam.First(checkKey) + if err != nil { + return nil, err + } + if fipInfo == nil { + return nil, fmt.Errorf("got nil ipInfo") + } + if !expectSubnet.Contains(fipInfo.IPInfo.IP.IP) { + return nil, fmt.Errorf("allocated ip %s is not in expect subnet %s", fipInfo.IPInfo.IP.IP.String(), + expectSubnet.String()) + } + return fipInfo, nil +} + +func TestReleaseIPOfFinishedPod(t *testing.T) { + for i, testCase := range []struct { + updatePodStatus func(pod *corev1.Pod) + }{ + {updatePodStatus: toFailedPod}, + {updatePodStatus: toSuccessPod}, + } { + pod := CreateStatefulSetPod("pod1-0", "ns1", nil) + podKey, _ := schedulerplugin_util.FormatKey(pod) + func() { + fipPlugin, stopChan, _ := createPluginTestNodes(t, pod) + fipPlugin.Run(stopChan) + defer func() { stopChan <- struct{}{} }() + fipInfo, err := checkBind(fipPlugin, pod, node3, podKey.KeyInDB, node3Subnet) + if err != nil { + t.Fatalf("case %d: %v", i, err) + } + testCase.updatePodStatus(pod) + if _, err := fipPlugin.Client.CoreV1().Pods(pod.Namespace).UpdateStatus(pod); err != nil { + t.Fatalf("case %d: %v", i, err) + } + if err := wait.Poll(time.Microsecond*10, time.Second*30, func() (done bool, err error) { + return checkIPKey(fipPlugin.ipam, fipInfo.FIP.IP.String(), "") == nil, nil + }); err != nil { + t.Fatalf("case %d: %v", i, err) + } + }() + } +} diff --git a/pkg/ipam/schedulerplugin/cloudprovider_test.go b/pkg/ipam/schedulerplugin/cloudprovider_test.go index 76d58d4b..b752e640 100644 --- a/pkg/ipam/schedulerplugin/cloudprovider_test.go +++ b/pkg/ipam/schedulerplugin/cloudprovider_test.go @@ -32,7 +32,7 @@ import ( func TestConcurrentBindUnbind(t *testing.T) { pod := CreateDeploymentPod("dp-xxx-yyy", "ns1", poolAnnotation("pool1")) podKey, _ := schedulerplugin_util.FormatKey(pod) - dp1 := createDeployment(pod.ObjectMeta, 1) + dp1 := CreateDeployment(pod.ObjectMeta, 1) plugin, stopChan, _ := createPluginTestNodes(t, pod, dp1) defer func() { stopChan <- struct{}{} }() cloudProvider := &fakeCloudProvider1{m: make(map[string]string)} diff --git a/pkg/ipam/schedulerplugin/deployment_test.go b/pkg/ipam/schedulerplugin/deployment_test.go index 6cbdfcb9..9cf290a8 100644 --- a/pkg/ipam/schedulerplugin/deployment_test.go +++ b/pkg/ipam/schedulerplugin/deployment_test.go @@ -39,7 +39,7 @@ func TestDpReleasePolicy(t *testing.T) { } { pod := CreateDeploymentPod("dp-xxx-yy", "ns1", testCase.annotations) keyObj, _ := util.FormatKey(pod) - dp := createDeployment(pod.ObjectMeta, testCase.replicas) + dp := CreateDeployment(pod.ObjectMeta, testCase.replicas) func() { fipPlugin, stopChan, _ := createPluginTestNodes(t, pod, dp) defer func() { stopChan <- struct{}{} }() diff --git a/pkg/ipam/schedulerplugin/filter.go b/pkg/ipam/schedulerplugin/filter.go new file mode 100644 index 00000000..8cec311f --- /dev/null +++ b/pkg/ipam/schedulerplugin/filter.go @@ -0,0 +1,144 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package schedulerplugin + +import ( + "fmt" + "net" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + glog "k8s.io/klog" + "tkestack.io/galaxy/pkg/api/galaxy/constant" + "tkestack.io/galaxy/pkg/api/k8s/schedulerapi" + "tkestack.io/galaxy/pkg/ipam/floatingip" + "tkestack.io/galaxy/pkg/ipam/metrics" + "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" +) + +// Filter marks nodes which have no available ips as FailedNodes +// If the given pod doesn't want floating IP, none failedNodes returns +func (p *FloatingIPPlugin) Filter(pod *corev1.Pod, nodes []corev1.Node) ([]corev1.Node, schedulerapi.FailedNodesMap, + error) { + start := time.Now() + failedNodesMap := schedulerapi.FailedNodesMap{} + if !p.hasResourceName(&pod.Spec) { + return nodes, failedNodesMap, nil + } + filteredNodes := []corev1.Node{} + defer p.lockPod(pod.Name, pod.Namespace)() + subnetSet, err := p.getSubnet(pod) + if err != nil { + return filteredNodes, failedNodesMap, err + } + for i := range nodes { + nodeName := nodes[i].Name + subnet, err := p.getNodeSubnet(&nodes[i]) + if err != nil { + failedNodesMap[nodes[i].Name] = err.Error() + continue + } + if subnetSet.Has(subnet.String()) { + filteredNodes = append(filteredNodes, nodes[i]) + } else { + failedNodesMap[nodeName] = "FloatingIPPlugin:NoFIPLeft" + } + } + if glog.V(5) { + nodeNames := make([]string, len(filteredNodes)) + for i := range filteredNodes { + nodeNames[i] = filteredNodes[i].Name + } + glog.V(5).Infof("filtered nodes %v failed nodes %v", nodeNames, failedNodesMap) + } + metrics.ScheduleLatency.WithLabelValues("filter").Observe(time.Since(start).Seconds()) + return filteredNodes, failedNodesMap, nil +} + +// #lizard forgives +func (p *FloatingIPPlugin) getSubnet(pod *corev1.Pod) (sets.String, error) { + keyObj, err := util.FormatKey(pod) + if err != nil { + return nil, err + } + cniArgs, err := getPodCniArgs(pod) + if err != nil { + return nil, err + } + // first check if exists an already allocated ip for this pod + subnets, err := p.ipam.NodeSubnetsByKeyAndIPRanges(keyObj.KeyInDB, cniArgs.RequestIPRange) + if err != nil { + return nil, fmt.Errorf("failed to query by key %s: %v", keyObj.KeyInDB, err) + } + if len(subnets) > 0 { + glog.V(3).Infof("%s already have an allocated ip in subnets %v", keyObj.KeyInDB, subnets) + return subnets, nil + } + policy := parseReleasePolicy(&pod.ObjectMeta) + if !keyObj.Deployment() && !keyObj.StatefulSet() && !keyObj.TApp() && policy != constant.ReleasePolicyPodDelete { + return nil, fmt.Errorf("policy %s not supported for non deployment/tapp/sts app", constant.PolicyStr(policy)) + } + var replicas int + var isPoolSizeDefined bool + if keyObj.Deployment() { + replicas, isPoolSizeDefined, err = p.getDpReplicas(keyObj) + if err != nil { + return nil, err + } + // Lock to make checking available subnets and allocating reserved ip atomic + defer p.LockDpPool(keyObj.PoolPrefix())() + } + subnetSet, reserve, err := p.getAvailableSubnet(keyObj, policy, replicas, isPoolSizeDefined, cniArgs.RequestIPRange) + if err != nil { + return nil, err + } + if (reserve || isPoolSizeDefined) && subnetSet.Len() > 0 { + // Since bind is in a different goroutine than filter in scheduler, we can't ensure this pod got binded + // before the next one got filtered to ensure max size of allocated ips. + // So we'd better do the allocate in filter for reserve situation. + reserveSubnet := subnetSet.List()[0] + subnetSet = sets.NewString(reserveSubnet) + if err := p.allocateDuringFilter(keyObj, reserve, isPoolSizeDefined, reserveSubnet, policy, + string(pod.UID)); err != nil { + return nil, err + } + } + return subnetSet, nil +} + +func (p *FloatingIPPlugin) allocateDuringFilter(keyObj *util.KeyObj, reserve, isPoolSizeDefined bool, + reserveSubnet string, policy constant.ReleasePolicy, uid string) error { + // we can't get nodename during filter, update attr on bind + attr := floatingip.Attr{Policy: policy, NodeName: "", Uid: uid} + if reserve { + if err := p.allocateInSubnetWithKey(keyObj.PoolPrefix(), keyObj.KeyInDB, reserveSubnet, attr, + "filter"); err != nil { + return err + } + } else if isPoolSizeDefined { + // if pool size defined and we got no reserved IP, we need to allocate IP from empty key + _, ipNet, err := net.ParseCIDR(reserveSubnet) + if err != nil { + return err + } + if err := p.allocateInSubnet(keyObj.KeyInDB, ipNet, attr, "filter"); err != nil { + return err + } + } + return nil +} diff --git a/pkg/ipam/schedulerplugin/filter_test.go b/pkg/ipam/schedulerplugin/filter_test.go new file mode 100644 index 00000000..f929274c --- /dev/null +++ b/pkg/ipam/schedulerplugin/filter_test.go @@ -0,0 +1,279 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package schedulerplugin + +import ( + "fmt" + "net" + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "tkestack.io/galaxy/pkg/api/galaxy/constant" + "tkestack.io/galaxy/pkg/api/k8s/schedulerapi" + "tkestack.io/galaxy/pkg/ipam/floatingip" + . "tkestack.io/galaxy/pkg/ipam/schedulerplugin/testing" + schedulerplugin_util "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" +) + +// #lizard forgives +func TestFilter(t *testing.T) { + fipPlugin, stopChan, nodes := createPluginTestNodes(t) + defer func() { stopChan <- struct{}{} }() + // pod has no floating ip resource name, filter should return all nodes + filtered, failed, err := fipPlugin.Filter(&corev1.Pod{ObjectMeta: v1.ObjectMeta{Name: "pod1", Namespace: "ns1"}}, nodes) + if err != nil { + t.Fatal(err) + } + if err := checkFilterResult(filtered, failed, []string{drainedNode, nodeHasNoIP, node3, node4}, []string{}); err != nil { + t.Fatal(err) + } + // a pod has floating ip resource name, filter should return nodes that has floating ips + if filtered, failed, err = fipPlugin.Filter(pod, nodes); err != nil { + t.Fatal(err) + } + if err := checkFilterResult(filtered, failed, []string{node3, node4}, []string{drainedNode, nodeHasNoIP}); err != nil { + t.Fatal(err) + } + // test filter for reserve situation + if err := fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), + floatingip.Attr{Policy: constant.ReleasePolicyPodDelete}); err != nil { + t.Fatal(err) + } + filtered, failed, err = fipPlugin.Filter(pod, nodes) + if err != nil { + t.Fatal(err) + } + if err := checkFilterResult(filtered, failed, []string{node4}, []string{drainedNode, nodeHasNoIP, node3}); err != nil { + t.Fatal(err) + } + // filter again on a new pod2, all good nodes should be filteredNodes + if filtered, failed, err = fipPlugin.Filter(CreateStatefulSetPod("pod2-1", "ns1", immutableAnnotation), nodes); err != nil { + t.Fatal(err) + } + if err := checkFilterResult(filtered, failed, []string{node3, node4}, []string{drainedNode, nodeHasNoIP}); err != nil { + t.Fatal(err) + } +} + +func TestFilterForPodWithoutRef(t *testing.T) { + fipPlugin, stopChan, nodes := createPluginTestNodes(t) + defer func() { stopChan <- struct{}{} }() + filtered, failed, err := fipPlugin.Filter(CreateSimplePod("pod1", "ns1", nil), nodes) + if err != nil { + t.Fatal(err) + } + if err := checkFilterResult(filtered, failed, []string{node3, node4}, []string{drainedNode, nodeHasNoIP}); err != nil { + t.Fatal(err) + } + if _, _, err = fipPlugin.Filter(CreateSimplePod("pod1", "ns1", immutableAnnotation), nodes); err == nil { + t.Fatalf("expect an error for non sts/deployment/tapp pod with policy immutable") + } +} + +// #lizard forgives +func TestFilterForDeployment(t *testing.T) { + deadPod := CreateDeploymentPod("dp-aaa-bbb", "ns1", immutableAnnotation) + pod := CreateDeploymentPod("dp-xxx-yyy", "ns1", immutableAnnotation) + dp := CreateDeployment(pod.ObjectMeta, 1) + fipPlugin, stopChan, nodes := createPluginTestNodes(t, pod, deadPod, dp) + defer func() { stopChan <- struct{}{} }() + // pre-allocate ip in filter for deployment pod + podKey, _ := schedulerplugin_util.FormatKey(pod) + deadPodKey, _ := schedulerplugin_util.FormatKey(deadPod) + cniArgs, err := fipPlugin.allocateIP(deadPodKey.KeyInDB, node3, deadPod) + if err != nil || len(cniArgs.Common.IPInfos) != 1 { + t.Fatal(err) + } + fip := cniArgs.Common.IPInfos[0] + // because deployment ip is allocated to deadPod, check if pod gets none available subnets + filtered, failed, err := fipPlugin.Filter(pod, nodes) + if err == nil || !strings.Contains(err.Error(), "wait for releasing") { + t.Fatal(err) + } + // because replicas = 1, ip will be reserved + if err := fipPlugin.unbind(deadPod); err != nil { + t.Fatal(err) + } + if filtered, failed, err = fipPlugin.Filter(pod, nodes); err != nil { + t.Fatal(err) + } + if err := checkFilterResult(filtered, failed, []string{node3}, []string{drainedNode, nodeHasNoIP, node4}); err != nil { + t.Fatal(err) + } + fip2, err := fipPlugin.ipam.First(podKey.KeyInDB) + if err != nil { + t.Fatal(err) + } else if fip.IP.String() != fip2.IPInfo.IP.String() { + t.Fatalf("allocate another ip, expect reserved one") + } + + pod.Annotations = neverAnnotation + deadPod.Annotations = immutableAnnotation + // when replicas = 0 and never release policy, ip will be reserved + *dp.Spec.Replicas = 0 + if err := fipPlugin.unbind(pod); err != nil { + t.Fatal(err) + } + *dp.Spec.Replicas = 1 + if filtered, failed, err = fipPlugin.Filter(deadPod, nodes); err != nil { + t.Fatal(err) + } + if err := checkFilterResult(filtered, failed, []string{node3}, []string{drainedNode, nodeHasNoIP, node4}); err != nil { + t.Fatal(err) + } + fip3, err := fipPlugin.ipam.First(deadPodKey.KeyInDB) + if err != nil { + t.Fatal(err) + } else if fip.IP.String() != fip3.IPInfo.IP.String() { + t.Fatalf("allocate another ip, expect reserved one") + } +} + +type filterCase struct { + testPod *corev1.Pod + expectErr error + expectFiltererd, expectFailed []string + preHook func() error + postHook func() error +} + +// #lizard forgives +func checkFilterCase(fipPlugin *FloatingIPPlugin, testCase filterCase, nodes []corev1.Node) error { + if testCase.preHook != nil { + if err := testCase.preHook(); err != nil { + return fmt.Errorf("preHook failed: %v", err) + } + } + filtered, failed, err := fipPlugin.Filter(testCase.testPod, nodes) + if !reflect.DeepEqual(err, testCase.expectErr) { + return fmt.Errorf("filter failed, expect err: %v, got: %v", testCase.expectErr, err) + } + if testCase.expectErr == nil && err != nil { + return fmt.Errorf("filter failed, expect nil err, got: %v", err) + } + if testCase.expectErr != nil && err == nil { + return fmt.Errorf("filter failed, expect none nil err %v, got nil err", testCase.expectErr) + } + if err := checkFilterResult(filtered, failed, testCase.expectFiltererd, testCase.expectFailed); err != nil { + return fmt.Errorf("checkFilterResult failed: %v", err) + } + if testCase.postHook != nil { + if err := testCase.postHook(); err != nil { + return fmt.Errorf("postHook failed: %v", err) + } + } + return nil +} + +// #lizard forgives +func TestFilterForDeploymentIPPool(t *testing.T) { + pod := CreateDeploymentPod("dp-xxx-yyy", "ns1", poolAnnotation("pool1")) + pod2 := CreateDeploymentPod("dp2-abc-def", "ns2", poolAnnotation("pool1")) + podKey, _ := schedulerplugin_util.FormatKey(pod) + pod2Key, _ := schedulerplugin_util.FormatKey(pod2) + dp1, dp2 := CreateDeployment(pod.ObjectMeta, 1), CreateDeployment(pod2.ObjectMeta, 1) + fipPlugin, stopChan, nodes := createPluginTestNodes(t, pod, pod2, dp1, dp2) + defer func() { stopChan <- struct{}{} }() + testCases := []filterCase{ + { + // test normal filter gets all good nodes + testPod: pod, expectFiltererd: []string{node3, node4}, expectFailed: []string{drainedNode, nodeHasNoIP}, + }, + { + // test bind gets the right key, i.e. dp_ns1_dp_dp-xxx-yyy, and filter gets reserved node + testPod: pod, expectFiltererd: []string{node4}, expectFailed: []string{drainedNode, nodeHasNoIP, node3}, + preHook: func() error { + return fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), + floatingip.Attr{Policy: constant.ReleasePolicyNever}) + }, + }, + { + // test unbind gets the right key, i.e. pool__pool1_, and filter on pod2 gets reserved node and key is updating to pod2, i.e. dp_ns1_dp2_dp2-abc-def + testPod: pod2, expectFiltererd: []string{node4}, expectFailed: []string{drainedNode, nodeHasNoIP, node3}, + preHook: func() error { + // because replicas = 1, ip will be reserved + if err := fipPlugin.unbind(pod); err != nil { + t.Fatal(err) + } + if err := checkIPKey(fipPlugin.ipam, "10.173.13.2", podKey.PoolPrefix()); err != nil { + t.Fatal(err) + } + return nil + }, + postHook: func() error { + if err := checkIPKey(fipPlugin.ipam, "10.173.13.2", pod2Key.KeyInDB); err != nil { + t.Fatal(err) + } + return nil + }, + }, + { + // test filter again on the same pool but different deployment pod and bind gets the right key, i.e. dp_ns1_dp_dp-xxx-yyy + // two pool deployment, deployment 1 gets enough ips, grow the pool size for deployment 2 + testPod: pod, expectFiltererd: []string{node3, node4}, expectFailed: []string{drainedNode, nodeHasNoIP}, + }, + } + for i := range testCases { + if err := checkFilterCase(fipPlugin, testCases[i], nodes); err != nil { + t.Fatalf("Case %d: %v", i, err) + } + } +} + +func checkFilterResult(realFilterd []corev1.Node, realFailed schedulerapi.FailedNodesMap, expectFiltererd, expectFailed []string) error { + if err := checkFiltered(realFilterd, expectFiltererd...); err != nil { + return err + } + if err := checkFailed(realFailed, expectFailed...); err != nil { + return err + } + return nil +} + +func checkFiltered(realFilterd []corev1.Node, filtererd ...string) error { + realNodeName := make([]string, len(realFilterd)) + for i := range realFilterd { + realNodeName[i] = realFilterd[i].Name + } + expect := sets.NewString(filtererd...) + if expect.Len() != len(realFilterd) { + return fmt.Errorf("filtered nodes missmatch, expect %v, real %v", expect, realNodeName) + } + for i := range realFilterd { + if !expect.Has(realFilterd[i].Name) { + return fmt.Errorf("filtered nodes missmatch, expect %v, real %v", expect, realNodeName) + } + } + return nil +} + +func checkFailed(realFailed schedulerapi.FailedNodesMap, failed ...string) error { + expect := sets.NewString(failed...) + if expect.Len() != len(realFailed) { + return fmt.Errorf("failed nodes missmatch, expect %v, real %v", expect, realFailed) + } + for nodeName := range realFailed { + if !expect.Has(nodeName) { + return fmt.Errorf("failed nodes missmatch, expect %v, real %v", expect, realFailed) + } + } + return nil +} diff --git a/pkg/ipam/schedulerplugin/floatingip_plugin.go b/pkg/ipam/schedulerplugin/floatingip_plugin.go index ee8d76d5..c30a09e9 100644 --- a/pkg/ipam/schedulerplugin/floatingip_plugin.go +++ b/pkg/ipam/schedulerplugin/floatingip_plugin.go @@ -17,7 +17,6 @@ package schedulerplugin import ( - "encoding/json" "fmt" "net" "sync" @@ -26,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" glog "k8s.io/klog" "k8s.io/utils/keymutex" @@ -34,10 +32,7 @@ import ( "tkestack.io/galaxy/pkg/api/galaxy/constant/utils" "tkestack.io/galaxy/pkg/api/k8s/schedulerapi" "tkestack.io/galaxy/pkg/ipam/cloudprovider" - "tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc" "tkestack.io/galaxy/pkg/ipam/floatingip" - "tkestack.io/galaxy/pkg/ipam/metrics" - "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" ) // FloatingIPPlugin Allocates Floating IP for deployments @@ -145,118 +140,6 @@ func (p *FloatingIPPlugin) updateConfigMap() (bool, error) { return true, nil } -// Filter marks nodes which have no available ips as FailedNodes -// If the given pod doesn't want floating IP, none failedNodes returns -func (p *FloatingIPPlugin) Filter(pod *corev1.Pod, nodes []corev1.Node) ([]corev1.Node, schedulerapi.FailedNodesMap, - error) { - start := time.Now() - failedNodesMap := schedulerapi.FailedNodesMap{} - if !p.hasResourceName(&pod.Spec) { - return nodes, failedNodesMap, nil - } - filteredNodes := []corev1.Node{} - defer p.lockPod(pod.Name, pod.Namespace)() - subnetSet, err := p.getSubnet(pod) - if err != nil { - return filteredNodes, failedNodesMap, err - } - for i := range nodes { - nodeName := nodes[i].Name - subnet, err := p.getNodeSubnet(&nodes[i]) - if err != nil { - failedNodesMap[nodes[i].Name] = err.Error() - continue - } - if subnetSet.Has(subnet.String()) { - filteredNodes = append(filteredNodes, nodes[i]) - } else { - failedNodesMap[nodeName] = "FloatingIPPlugin:NoFIPLeft" - } - } - if glog.V(5) { - nodeNames := make([]string, len(filteredNodes)) - for i := range filteredNodes { - nodeNames[i] = filteredNodes[i].Name - } - glog.V(5).Infof("filtered nodes %v failed nodes %v", nodeNames, failedNodesMap) - } - metrics.ScheduleLatency.WithLabelValues("filter").Observe(time.Since(start).Seconds()) - return filteredNodes, failedNodesMap, nil -} - -// #lizard forgives -func (p *FloatingIPPlugin) getSubnet(pod *corev1.Pod) (sets.String, error) { - keyObj, err := util.FormatKey(pod) - if err != nil { - return nil, err - } - cniArgs, err := getPodCniArgs(pod) - if err != nil { - return nil, err - } - // first check if exists an already allocated ip for this pod - subnets, err := p.ipam.NodeSubnetsByKeyAndIPRanges(keyObj.KeyInDB, cniArgs.RequestIPRange) - if err != nil { - return nil, fmt.Errorf("failed to query by key %s: %v", keyObj.KeyInDB, err) - } - if len(subnets) > 0 { - glog.V(3).Infof("%s already have an allocated ip in subnets %v", keyObj.KeyInDB, subnets) - return subnets, nil - } - policy := parseReleasePolicy(&pod.ObjectMeta) - if !keyObj.Deployment() && !keyObj.StatefulSet() && !keyObj.TApp() && policy != constant.ReleasePolicyPodDelete { - return nil, fmt.Errorf("policy %s not supported for non deployment/tapp/sts app", constant.PolicyStr(policy)) - } - var replicas int - var isPoolSizeDefined bool - if keyObj.Deployment() { - replicas, isPoolSizeDefined, err = p.getDpReplicas(keyObj) - if err != nil { - return nil, err - } - // Lock to make checking available subnets and allocating reserved ip atomic - defer p.LockDpPool(keyObj.PoolPrefix())() - } - subnetSet, reserve, err := p.getAvailableSubnet(keyObj, policy, replicas, isPoolSizeDefined, cniArgs.RequestIPRange) - if err != nil { - return nil, err - } - if (reserve || isPoolSizeDefined) && subnetSet.Len() > 0 { - // Since bind is in a different goroutine than filter in scheduler, we can't ensure this pod got binded - // before the next one got filtered to ensure max size of allocated ips. - // So we'd better do the allocate in filter for reserve situation. - reserveSubnet := subnetSet.List()[0] - subnetSet = sets.NewString(reserveSubnet) - if err := p.allocateDuringFilter(keyObj, reserve, isPoolSizeDefined, reserveSubnet, policy, - string(pod.UID)); err != nil { - return nil, err - } - } - return subnetSet, nil -} - -func (p *FloatingIPPlugin) allocateDuringFilter(keyObj *util.KeyObj, reserve, isPoolSizeDefined bool, - reserveSubnet string, policy constant.ReleasePolicy, uid string) error { - // we can't get nodename during filter, update attr on bind - attr := floatingip.Attr{Policy: policy, NodeName: "", Uid: uid} - if reserve { - if err := p.allocateInSubnetWithKey(keyObj.PoolPrefix(), keyObj.KeyInDB, reserveSubnet, attr, - "filter"); err != nil { - return err - } - } else if isPoolSizeDefined { - // if pool size defined and we got no reserved IP, we need to allocate IP from empty key - _, ipNet, err := net.ParseCIDR(reserveSubnet) - if err != nil { - return err - } - if err := p.allocateInSubnet(keyObj.KeyInDB, ipNet, attr, "filter"); err != nil { - return err - } - } - return nil -} - // Prioritize can score each node, currently it does nothing func (p *FloatingIPPlugin) Prioritize(pod *corev1.Pod, nodes []corev1.Node) (*schedulerapi.HostPriorityList, error) { list := &schedulerapi.HostPriorityList{} @@ -267,168 +150,6 @@ func (p *FloatingIPPlugin) Prioritize(pod *corev1.Pod, nodes []corev1.Node) (*sc return list, nil } -func (p *FloatingIPPlugin) allocateIP(key string, nodeName string, pod *corev1.Pod) (*constant.CniArgs, error) { - var how string - cniArgs, err := getPodCniArgs(pod) - if err != nil { - return nil, err - } - ipInfos, err := p.ipam.ByKeyAndIPRanges(key, cniArgs.RequestIPRange) - if err != nil { - return nil, fmt.Errorf("failed to query floating ip by key %s: %v", key, err) - } - started := time.Now() - policy := parseReleasePolicy(&pod.ObjectMeta) - attr := floatingip.Attr{Policy: policy, NodeName: nodeName, Uid: string(pod.UID)} - if len(ipInfos) != 0 { - how = "reused" - for _, ipInfo := range ipInfos { - // check if uid missmatch, if we delete a statfulset/tapp and creates a same name statfulset/tapp immediately, - // galaxy-ipam may receive bind event for new pod early than deleting event for old pod - if ipInfo.FIP.PodUid != "" && ipInfo.FIP.PodUid != string(pod.GetUID()) { - return nil, fmt.Errorf("waiting for delete event of %s before reuse this ip", key) - } - } - } else { - subnet, err := p.queryNodeSubnet(nodeName) - if err != nil { - return nil, err - } - if _, err := p.ipam.AllocateInSubnetsAndIPRange(key, subnet, cniArgs.RequestIPRange, attr); err != nil { - return nil, err - } - how = "allocated" - ipInfos, err = p.ipam.ByKeyAndIPRanges(key, cniArgs.RequestIPRange) - if err != nil { - return nil, fmt.Errorf("failed to query floating ip by key %s: %v", key, err) - } - if len(ipInfos) == 0 { - return nil, fmt.Errorf("nil floating ip for key %s: %v", key, err) - } - } - for _, ipInfo := range ipInfos { - glog.Infof("AssignIP nodeName %s, ip %s, key %s", nodeName, ipInfo.IPInfo.IP.IP.String(), key) - if err := p.cloudProviderAssignIP(&rpc.AssignIPRequest{ - NodeName: nodeName, - IPAddress: ipInfo.IPInfo.IP.IP.String(), - }); err != nil { - // do not rollback allocated ip - return nil, fmt.Errorf("failed to assign ip %s to %s: %v", ipInfo.IPInfo.IP.IP.String(), key, err) - } - if how == "reused" { - glog.Infof("pod %s reused %s, updating attr to %v", key, ipInfo.IPInfo.IP.String(), attr) - if err := p.ipam.UpdateAttr(key, ipInfo.IPInfo.IP.IP, attr); err != nil { - return nil, fmt.Errorf("failed to update floating ip release policy: %v", err) - } - } - } - var ips []string - var ret []constant.IPInfo - for _, ipInfo := range ipInfos { - ips = append(ips, ipInfo.IPInfo.IP.String()) - ret = append(ret, ipInfo.IPInfo) - } - glog.Infof("started at %d %s ips %s, attr %v for %s", started.UnixNano(), how, ips, attr, key) - cniArgs.Common.IPInfos = ret - return &cniArgs, nil -} - -// Bind binds a new floatingip or reuse an old one to pod -func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error { - start := time.Now() - pod, err := p.PluginFactoryArgs.PodLister.Pods(args.PodNamespace).Get(args.PodName) - if err != nil { - return fmt.Errorf("failed to find pod %s: %w", util.Join(args.PodName, args.PodNamespace), err) - } - if !p.hasResourceName(&pod.Spec) { - // we will config extender resources which ensures pod which doesn't want floatingip won't be sent to plugin - // see https://github.com/kubernetes/kubernetes/pull/60332 - return fmt.Errorf("pod which doesn't want floatingip have been sent to plugin") - } - defer p.lockPod(pod.Name, pod.Namespace)() - keyObj, err := util.FormatKey(pod) - if err != nil { - return err - } - cniArgs, err := p.allocateIP(keyObj.KeyInDB, args.Node, pod) - if err != nil { - return err - } - data, err := json.Marshal(cniArgs) - if err != nil { - return fmt.Errorf("marshal cni args %v: %v", *cniArgs, err) - } - bindAnnotation := map[string]string{constant.ExtendedCNIArgsAnnotation: string(data)} - var err1 error - if err := wait.PollImmediate(time.Millisecond*500, 3*time.Second, func() (bool, error) { - // It's the extender's response to bind pods to nodes since it is a binder - if err := p.Client.CoreV1().Pods(args.PodNamespace).Bind(&corev1.Binding{ - ObjectMeta: v1.ObjectMeta{Namespace: args.PodNamespace, Name: args.PodName, UID: args.PodUID, - Annotations: bindAnnotation}, - Target: corev1.ObjectReference{ - Kind: "Node", - Name: args.Node, - }, - }); err != nil { - err1 = err - if isPodNotFoundError(err) { - // break retry if pod no longer exists - return false, err - } - return false, nil - } - glog.Infof("bind pod %s to %s with ip %v", keyObj.KeyInDB, args.Node, - bindAnnotation[constant.ExtendedCNIArgsAnnotation]) - return true, nil - }); err != nil { - if isPodNotFoundError(err1) { - glog.Infof("binding returns not found for pod %s, putting it into unreleased chan", keyObj.KeyInDB) - // attach ip annotation - p.unreleased <- &releaseEvent{pod: pod} - } - // If fails to update, depending on resync to update - return fmt.Errorf("update pod %s: %w", keyObj.KeyInDB, err1) - } - metrics.ScheduleLatency.WithLabelValues("bind").Observe(time.Since(start).Seconds()) - return nil -} - -func isPodNotFoundError(err error) bool { - return apierrors.IsNotFound(err) -} - -// unbind release ip from pod -func (p *FloatingIPPlugin) unbind(pod *corev1.Pod) error { - defer p.lockPod(pod.Name, pod.Namespace)() - glog.V(3).Infof("handle unbind pod %s", pod.Name) - keyObj, err := util.FormatKey(pod) - if err != nil { - return err - } - key := keyObj.KeyInDB - if p.cloudProvider != nil { - ipInfos, err := p.ipam.ByKeyAndIPRanges(key, nil) - if err != nil { - return fmt.Errorf("query floating ip by key %s: %v", key, err) - } - for _, ipInfo := range ipInfos { - ipStr := ipInfo.IPInfo.IP.IP.String() - glog.Infof("UnAssignIP nodeName %s, ip %s, key %s", ipInfo.FIP.NodeName, ipStr, key) - if err = p.cloudProviderUnAssignIP(&rpc.UnAssignIPRequest{ - NodeName: ipInfo.FIP.NodeName, - IPAddress: ipStr, - }); err != nil { - return fmt.Errorf("failed to unassign ip %s from %s: %v", ipStr, key, err) - } - } - } - policy := parseReleasePolicy(&pod.ObjectMeta) - if keyObj.Deployment() { - return p.unbindDpPod(keyObj, policy, "during unbinding pod") - } - return p.unbindNoneDpPod(keyObj, policy, "during unbinding pod") -} - // hasResourceName checks if the podspec has floatingip resource name func (p *FloatingIPPlugin) hasResourceName(spec *corev1.PodSpec) bool { return utils.WantENIIP(spec) diff --git a/pkg/ipam/schedulerplugin/floatingip_plugin_test.go b/pkg/ipam/schedulerplugin/floatingip_plugin_test.go index e65fe090..dabf527f 100644 --- a/pkg/ipam/schedulerplugin/floatingip_plugin_test.go +++ b/pkg/ipam/schedulerplugin/floatingip_plugin_test.go @@ -20,27 +20,19 @@ import ( "encoding/json" "fmt" "net" - "reflect" - "strings" "testing" "time" - appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" extensionClient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" coreInformer "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes/fake" - fakeV1 "k8s.io/client-go/kubernetes/typed/core/v1/fake" "tkestack.io/galaxy/pkg/api/galaxy/constant" - "tkestack.io/galaxy/pkg/api/k8s/schedulerapi" fakeGalaxyCli "tkestack.io/galaxy/pkg/ipam/client/clientset/versioned/fake" crdInformer "tkestack.io/galaxy/pkg/ipam/client/informers/externalversions" - "tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc" "tkestack.io/galaxy/pkg/ipam/floatingip" . "tkestack.io/galaxy/pkg/ipam/schedulerplugin/testing" schedulerplugin_util "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" @@ -73,10 +65,10 @@ var ( func createPluginTestNodes(t *testing.T, objs ...runtime.Object) (*FloatingIPPlugin, chan struct{}, []corev1.Node) { nodes := []corev1.Node{ - createNode(drainedNode, nil, "10.180.1.3"), // no floating ip left on this node - createNode(nodeHasNoIP, nil, "10.48.28.2"), // no floating ip configured for this node - createNode(node3, nil, "10.49.27.3"), // good node - createNode(node4, nil, "10.173.13.4"), // good node + CreateNode(drainedNode, nil, "10.180.1.3"), // no floating ip left on this node + CreateNode(nodeHasNoIP, nil, "10.48.28.2"), // no floating ip configured for this node + CreateNode(node3, nil, "10.49.27.3"), // good node + CreateNode(node4, nil, "10.173.13.4"), // good node } allObjs := append([]runtime.Object{&nodes[0], &nodes[1], &nodes[2], &nodes[3]}, objs...) fipPlugin, stopChan := createPlugin(t, allObjs...) @@ -97,94 +89,6 @@ func createPlugin(t *testing.T, objs ...runtime.Object) (*FloatingIPPlugin, chan return fipPlugin, stopChan } -// #lizard forgives -func TestFilter(t *testing.T) { - fipPlugin, stopChan, nodes := createPluginTestNodes(t) - defer func() { stopChan <- struct{}{} }() - // pod has no floating ip resource name, filter should return all nodes - filtered, failed, err := fipPlugin.Filter(&corev1.Pod{ObjectMeta: v1.ObjectMeta{Name: "pod1", Namespace: "ns1"}}, nodes) - if err != nil { - t.Fatal(err) - } - if err := checkFilterResult(filtered, failed, []string{drainedNode, nodeHasNoIP, node3, node4}, []string{}); err != nil { - t.Fatal(err) - } - // a pod has floating ip resource name, filter should return nodes that has floating ips - if filtered, failed, err = fipPlugin.Filter(pod, nodes); err != nil { - t.Fatal(err) - } - if err := checkFilterResult(filtered, failed, []string{node3, node4}, []string{drainedNode, nodeHasNoIP}); err != nil { - t.Fatal(err) - } - // test filter for reserve situation - if err := fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), - floatingip.Attr{Policy: constant.ReleasePolicyPodDelete}); err != nil { - t.Fatal(err) - } - filtered, failed, err = fipPlugin.Filter(pod, nodes) - if err != nil { - t.Fatal(err) - } - if err := checkFilterResult(filtered, failed, []string{node4}, []string{drainedNode, nodeHasNoIP, node3}); err != nil { - t.Fatal(err) - } - // filter again on a new pod2, all good nodes should be filteredNodes - if filtered, failed, err = fipPlugin.Filter(CreateStatefulSetPod("pod2-1", "ns1", immutableAnnotation), nodes); err != nil { - t.Fatal(err) - } - if err := checkFilterResult(filtered, failed, []string{node3, node4}, []string{drainedNode, nodeHasNoIP}); err != nil { - t.Fatal(err) - } -} - -func TestFilterForPodWithoutRef(t *testing.T) { - fipPlugin, stopChan, nodes := createPluginTestNodes(t) - defer func() { stopChan <- struct{}{} }() - filtered, failed, err := fipPlugin.Filter(CreateSimplePod("pod1", "ns1", nil), nodes) - if err != nil { - t.Fatal(err) - } - if err := checkFilterResult(filtered, failed, []string{node3, node4}, []string{drainedNode, nodeHasNoIP}); err != nil { - t.Fatal(err) - } - if _, _, err = fipPlugin.Filter(CreateSimplePod("pod1", "ns1", immutableAnnotation), nodes); err == nil { - t.Fatalf("expect an error for non sts/deployment/tapp pod with policy immutable") - } -} - -func TestAllocateIP(t *testing.T) { - fipPlugin, stopChan, _ := createPluginTestNodes(t) - defer func() { stopChan <- struct{}{} }() - - if err := fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), - floatingip.Attr{Policy: constant.ReleasePolicyPodDelete}); err != nil { - t.Fatal(err) - } - // check update from ReleasePolicyPodDelete to ReleasePolicyImmutable - pod.Spec.NodeName = node4 - pod.SetUID("pod-xx-1") - cniArgs, err := fipPlugin.allocateIP(podKey.KeyInDB, pod.Spec.NodeName, pod) - if err != nil || len(cniArgs.Common.IPInfos) != 1 { - t.Fatal(err) - } - if cniArgs.Common.IPInfos[0].IP.String() != "10.173.13.2/24" { - t.Fatal(cniArgs.Common.IPInfos[0]) - } - fip, err := fipPlugin.ipam.First(podKey.KeyInDB) - if err != nil { - t.Fatal(err) - } - if fip.FIP.Policy != uint16(constant.ReleasePolicyImmutable) { - t.Fatal(fip.FIP.Policy) - } - if fip.FIP.NodeName != node4 { - t.Fatal(fip.FIP.NodeName) - } - if fip.FIP.PodUid != string(pod.UID) { - t.Fatal(fip.FIP.PodUid) - } -} - func TestUpdatePod(t *testing.T) { fipPlugin, stopChan, _ := createPluginTestNodes(t) defer func() { stopChan <- struct{}{} }() @@ -228,247 +132,11 @@ func TestReleaseIP(t *testing.T) { } } -// #lizard forgives -func TestFilterForDeployment(t *testing.T) { - deadPod := CreateDeploymentPod("dp-aaa-bbb", "ns1", immutableAnnotation) - pod := CreateDeploymentPod("dp-xxx-yyy", "ns1", immutableAnnotation) - dp := createDeployment(pod.ObjectMeta, 1) - fipPlugin, stopChan, nodes := createPluginTestNodes(t, pod, deadPod, dp) - defer func() { stopChan <- struct{}{} }() - // pre-allocate ip in filter for deployment pod - podKey, _ := schedulerplugin_util.FormatKey(pod) - deadPodKey, _ := schedulerplugin_util.FormatKey(deadPod) - cniArgs, err := fipPlugin.allocateIP(deadPodKey.KeyInDB, node3, deadPod) - if err != nil || len(cniArgs.Common.IPInfos) != 1 { - t.Fatal(err) - } - fip := cniArgs.Common.IPInfos[0] - // because deployment ip is allocated to deadPod, check if pod gets none available subnets - filtered, failed, err := fipPlugin.Filter(pod, nodes) - if err == nil || !strings.Contains(err.Error(), "wait for releasing") { - t.Fatal(err) - } - // because replicas = 1, ip will be reserved - if err := fipPlugin.unbind(deadPod); err != nil { - t.Fatal(err) - } - if filtered, failed, err = fipPlugin.Filter(pod, nodes); err != nil { - t.Fatal(err) - } - if err := checkFilterResult(filtered, failed, []string{node3}, []string{drainedNode, nodeHasNoIP, node4}); err != nil { - t.Fatal(err) - } - fip2, err := fipPlugin.ipam.First(podKey.KeyInDB) - if err != nil { - t.Fatal(err) - } else if fip.IP.String() != fip2.IPInfo.IP.String() { - t.Fatalf("allocate another ip, expect reserved one") - } - - pod.Annotations = neverAnnotation - deadPod.Annotations = immutableAnnotation - // when replicas = 0 and never release policy, ip will be reserved - *dp.Spec.Replicas = 0 - if err := fipPlugin.unbind(pod); err != nil { - t.Fatal(err) - } - *dp.Spec.Replicas = 1 - if filtered, failed, err = fipPlugin.Filter(deadPod, nodes); err != nil { - t.Fatal(err) - } - if err := checkFilterResult(filtered, failed, []string{node3}, []string{drainedNode, nodeHasNoIP, node4}); err != nil { - t.Fatal(err) - } - fip3, err := fipPlugin.ipam.First(deadPodKey.KeyInDB) - if err != nil { - t.Fatal(err) - } else if fip.IP.String() != fip3.IPInfo.IP.String() { - t.Fatalf("allocate another ip, expect reserved one") - } -} - func poolAnnotation(poolName string) map[string]string { return map[string]string{constant.IPPoolAnnotation: poolName} } -func createDeployment(podMeta v1.ObjectMeta, replicas int32) *appv1.Deployment { - parts := strings.Split(podMeta.OwnerReferences[0].Name, "-") - return &appv1.Deployment{ - ObjectMeta: v1.ObjectMeta{ - Name: strings.Join(parts[:len(parts)-1], "-"), - Namespace: podMeta.Namespace, - Labels: podMeta.Labels, - }, - Spec: appv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: podMeta, - }, - Replicas: &replicas, - Selector: &v1.LabelSelector{ - MatchLabels: podMeta.GetLabels(), - }, - }, - } -} - -func CreateStatefulSet(podMeta v1.ObjectMeta, replicas int32) *appv1.StatefulSet { - return &appv1.StatefulSet{ - ObjectMeta: v1.ObjectMeta{ - Name: podMeta.OwnerReferences[0].Name, - Namespace: podMeta.GetNamespace(), - Labels: podMeta.GetLabels(), - }, - Spec: appv1.StatefulSetSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: podMeta, - }, - Replicas: &replicas, - Selector: &v1.LabelSelector{ - MatchLabels: podMeta.GetLabels(), - }, - }, - } -} - -type filterCase struct { - testPod *corev1.Pod - expectErr error - expectFiltererd, expectFailed []string - preHook func() error - postHook func() error -} - -// #lizard forgives -func checkFilterCase(fipPlugin *FloatingIPPlugin, testCase filterCase, nodes []corev1.Node) error { - if testCase.preHook != nil { - if err := testCase.preHook(); err != nil { - return fmt.Errorf("preHook failed: %v", err) - } - } - filtered, failed, err := fipPlugin.Filter(testCase.testPod, nodes) - if !reflect.DeepEqual(err, testCase.expectErr) { - return fmt.Errorf("filter failed, expect err: %v, got: %v", testCase.expectErr, err) - } - if testCase.expectErr == nil && err != nil { - return fmt.Errorf("filter failed, expect nil err, got: %v", err) - } - if testCase.expectErr != nil && err == nil { - return fmt.Errorf("filter failed, expect none nil err %v, got nil err", testCase.expectErr) - } - if err := checkFilterResult(filtered, failed, testCase.expectFiltererd, testCase.expectFailed); err != nil { - return fmt.Errorf("checkFilterResult failed: %v", err) - } - if testCase.postHook != nil { - if err := testCase.postHook(); err != nil { - return fmt.Errorf("postHook failed: %v", err) - } - } - return nil -} - -// #lizard forgives -func TestFilterForDeploymentIPPool(t *testing.T) { - pod := CreateDeploymentPod("dp-xxx-yyy", "ns1", poolAnnotation("pool1")) - pod2 := CreateDeploymentPod("dp2-abc-def", "ns2", poolAnnotation("pool1")) - podKey, _ := schedulerplugin_util.FormatKey(pod) - pod2Key, _ := schedulerplugin_util.FormatKey(pod2) - dp1, dp2 := createDeployment(pod.ObjectMeta, 1), createDeployment(pod2.ObjectMeta, 1) - fipPlugin, stopChan, nodes := createPluginTestNodes(t, pod, pod2, dp1, dp2) - defer func() { stopChan <- struct{}{} }() - testCases := []filterCase{ - { - // test normal filter gets all good nodes - testPod: pod, expectFiltererd: []string{node3, node4}, expectFailed: []string{drainedNode, nodeHasNoIP}, - }, - { - // test bind gets the right key, i.e. dp_ns1_dp_dp-xxx-yyy, and filter gets reserved node - testPod: pod, expectFiltererd: []string{node4}, expectFailed: []string{drainedNode, nodeHasNoIP, node3}, - preHook: func() error { - return fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), - floatingip.Attr{Policy: constant.ReleasePolicyNever}) - }, - }, - { - // test unbind gets the right key, i.e. pool__pool1_, and filter on pod2 gets reserved node and key is updating to pod2, i.e. dp_ns1_dp2_dp2-abc-def - testPod: pod2, expectFiltererd: []string{node4}, expectFailed: []string{drainedNode, nodeHasNoIP, node3}, - preHook: func() error { - // because replicas = 1, ip will be reserved - if err := fipPlugin.unbind(pod); err != nil { - t.Fatal(err) - } - if err := checkIPKey(fipPlugin.ipam, "10.173.13.2", podKey.PoolPrefix()); err != nil { - t.Fatal(err) - } - return nil - }, - postHook: func() error { - if err := checkIPKey(fipPlugin.ipam, "10.173.13.2", pod2Key.KeyInDB); err != nil { - t.Fatal(err) - } - return nil - }, - }, - { - // test filter again on the same pool but different deployment pod and bind gets the right key, i.e. dp_ns1_dp_dp-xxx-yyy - // two pool deployment, deployment 1 gets enough ips, grow the pool size for deployment 2 - testPod: pod, expectFiltererd: []string{node3, node4}, expectFailed: []string{drainedNode, nodeHasNoIP}, - }, - } - for i := range testCases { - if err := checkFilterCase(fipPlugin, testCases[i], nodes); err != nil { - t.Fatalf("Case %d: %v", i, err) - } - } -} - -func checkFilterResult(realFilterd []corev1.Node, realFailed schedulerapi.FailedNodesMap, expectFiltererd, expectFailed []string) error { - if err := checkFiltered(realFilterd, expectFiltererd...); err != nil { - return err - } - if err := checkFailed(realFailed, expectFailed...); err != nil { - return err - } - return nil -} - -func checkFiltered(realFilterd []corev1.Node, filtererd ...string) error { - realNodeName := make([]string, len(realFilterd)) - for i := range realFilterd { - realNodeName[i] = realFilterd[i].Name - } - expect := sets.NewString(filtererd...) - if expect.Len() != len(realFilterd) { - return fmt.Errorf("filtered nodes missmatch, expect %v, real %v", expect, realNodeName) - } - for i := range realFilterd { - if !expect.Has(realFilterd[i].Name) { - return fmt.Errorf("filtered nodes missmatch, expect %v, real %v", expect, realNodeName) - } - } - return nil -} - -func checkFailed(realFailed schedulerapi.FailedNodesMap, failed ...string) error { - expect := sets.NewString(failed...) - if expect.Len() != len(realFailed) { - return fmt.Errorf("failed nodes missmatch, expect %v, real %v", expect, realFailed) - } - for nodeName := range realFailed { - if !expect.Has(nodeName) { - return fmt.Errorf("failed nodes missmatch, expect %v, real %v", expect, realFailed) - } - } - return nil -} - -func createNode(name string, labels map[string]string, address string) corev1.Node { - return corev1.Node{ - ObjectMeta: v1.ObjectMeta{Name: name, Labels: labels}, - Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: address}}}, - } -} - -func createPluginFactoryArgs(t *testing.T, objs ...runtime.Object) (*PluginFactoryArgs, coreInformer.PodInformer, chan struct{}) { +func createPluginFactoryArgs(objs ...runtime.Object) (*PluginFactoryArgs, coreInformer.PodInformer, chan struct{}) { galaxyCli := fakeGalaxyCli.NewSimpleClientset() crdInformerFactory := crdInformer.NewSharedInformerFactory(galaxyCli, 0) poolInformer := crdInformerFactory.Galaxy().V1alpha1().Pools() @@ -505,7 +173,7 @@ func createPluginFactoryArgs(t *testing.T, objs ...runtime.Object) (*PluginFacto } func newPlugin(t *testing.T, conf Conf, objs ...runtime.Object) (*FloatingIPPlugin, chan struct{}) { - pluginArgs, podInformer, stopChan := createPluginFactoryArgs(t, objs...) + pluginArgs, podInformer, stopChan := createPluginFactoryArgs(objs...) fipPlugin, err := NewFloatingIPPlugin(conf, pluginArgs) if err != nil { t.Fatal(err) @@ -539,39 +207,6 @@ func TestLoadConfigMap(t *testing.T) { } } -func TestBind(t *testing.T) { - fipPlugin, stopChan, _ := createPluginTestNodes(t, pod) - defer func() { stopChan <- struct{}{} }() - fipInfo, err := checkBind(fipPlugin, pod, node3, podKey.KeyInDB, node3Subnet) - if err != nil { - t.Fatalf("checkBind error %v", err) - } - fakePods := fipPlugin.PluginFactoryArgs.Client.CoreV1().Pods(pod.Namespace).(*fakeV1.FakePods) - - actualBinding, err := fakePods.GetBinding(pod.GetName()) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - return - } - str, err := constant.MarshalCniArgs([]constant.IPInfo{fipInfo.IPInfo}) - if err != nil { - t.Fatal(err) - } - expect := &corev1.Binding{ - ObjectMeta: v1.ObjectMeta{ - Namespace: pod.Namespace, Name: pod.Name, - Annotations: map[string]string{ - constant.ExtendedCNIArgsAnnotation: str}}, - Target: corev1.ObjectReference{ - Kind: "Node", - Name: node3, - }, - } - if !reflect.DeepEqual(expect, actualBinding) { - t.Fatalf("Binding did not match expectation, expect %v, actual %v", expect, actualBinding) - } -} - func TestParseReleasePolicy(t *testing.T) { testCases := []struct { meta *v1.ObjectMeta @@ -607,83 +242,6 @@ func TestParseReleasePolicy(t *testing.T) { } } -type fakeCloudProvider struct { - expectIP string - expectNode string - invokedAssignIP bool - invokedUnAssignIP bool -} - -func (f *fakeCloudProvider) AssignIP(in *rpc.AssignIPRequest) (*rpc.AssignIPReply, error) { - f.invokedAssignIP = true - if in == nil { - return nil, fmt.Errorf("nil request") - } - if in.IPAddress != f.expectIP { - return nil, fmt.Errorf("expect ip %s, got %s", f.expectIP, in.IPAddress) - } - if in.NodeName != f.expectNode { - return nil, fmt.Errorf("expect node name %s, got %s", f.expectNode, in.NodeName) - } - return &rpc.AssignIPReply{Success: true}, nil -} - -func (f *fakeCloudProvider) UnAssignIP(in *rpc.UnAssignIPRequest) (*rpc.UnAssignIPReply, error) { - f.invokedUnAssignIP = true - if in == nil { - return nil, fmt.Errorf("nil request") - } - if in.IPAddress != f.expectIP { - return nil, fmt.Errorf("expect ip %s, got %s", f.expectIP, in.IPAddress) - } - if in.NodeName != f.expectNode { - return nil, fmt.Errorf("expect node name %s, got %s", f.expectNode, in.NodeName) - } - return &rpc.UnAssignIPReply{Success: true}, nil -} - -// #lizard forgives -func TestUnBind(t *testing.T) { - pod1 := CreateStatefulSetPod("pod1-1", "demo", map[string]string{}) - keyObj, _ := schedulerplugin_util.FormatKey(pod1) - fipPlugin, stopChan, _ := createPluginTestNodes(t, pod1) - defer func() { stopChan <- struct{}{} }() - fipPlugin.cloudProvider = &fakeCloudProvider{} - // if a pod has not got cni args annotation, unbind should return nil - if err := fipPlugin.unbind(pod1); err != nil { - t.Fatal(err) - } - // if a pod has got bad cni args annotation, - // unbind should return nil because we got binded ip from store instead of annotation - pod1.Annotations[constant.ExtendedCNIArgsAnnotation] = "fff" - if err := fipPlugin.unbind(pod1); err != nil { - t.Fatal(err) - } - // bind before testing normal unbind - expectIP := net.ParseIP("10.49.27.205") - if err := drainNode(fipPlugin, node3Subnet, expectIP); err != nil { - t.Fatal(err) - } - fakeCP := &fakeCloudProvider{expectIP: expectIP.String(), expectNode: node3} - fipPlugin.cloudProvider = fakeCP - fipInfo, err := checkBind(fipPlugin, pod1, node3, keyObj.KeyInDB, node3Subnet) - if err != nil { - t.Fatal(err) - } - str, err := constant.MarshalCniArgs([]constant.IPInfo{fipInfo.IPInfo}) - if err != nil { - t.Fatal(err) - } - pod1.Annotations[constant.ExtendedCNIArgsAnnotation] = str - pod1.Spec.NodeName = node3 - if err := fipPlugin.unbind(pod1); err != nil { - t.Fatal(err) - } - if !fakeCP.invokedAssignIP || !fakeCP.invokedUnAssignIP { - t.Fatal() - } -} - func drainNode(fipPlugin *FloatingIPPlugin, subnet *net.IPNet, except net.IP) error { for { if _, err := fipPlugin.ipam.AllocateInSubnet("ns_notexistpod", subnet, @@ -700,54 +258,6 @@ func drainNode(fipPlugin *FloatingIPPlugin, subnet *net.IPNet, except net.IP) er return nil } -func TestUnBindImmutablePod(t *testing.T) { - pod = CreateStatefulSetPodWithLabels("sts1-0", "ns1", map[string]string{"app": "sts1"}, immutableAnnotation) - podKey, _ = schedulerplugin_util.FormatKey(pod) - fipPlugin, stopChan, _ := createPluginTestNodes(t, pod, CreateStatefulSet(pod.ObjectMeta, 1)) - defer func() { stopChan <- struct{}{} }() - if err := fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), - floatingip.Attr{Policy: constant.ReleasePolicyImmutable}); err != nil { - t.Fatal(err) - } - // unbind the pod, check ip should be reserved, because pod has is immutable - if err := fipPlugin.unbind(pod); err != nil { - t.Fatal(err) - } - if err := checkIPKey(fipPlugin.ipam, "10.173.13.2", podKey.KeyInDB); err != nil { - t.Fatal(err) - } -} - -// #lizard forgives -func TestAllocateRecentIPs(t *testing.T) { - pod := CreateDeploymentPod("dp-xxx-yyy", "ns1", poolAnnotation("pool1")) - dp := createDeployment(pod.ObjectMeta, 1) - fipPlugin, stopChan, nodes := createPluginTestNodes(t, pod, dp) - defer func() { stopChan <- struct{}{} }() - podKey, _ := schedulerplugin_util.FormatKey(pod) - if err := fipPlugin.ipam.AllocateSpecificIP(podKey.PoolPrefix(), net.ParseIP("10.49.27.205"), - floatingip.Attr{Policy: constant.ReleasePolicyPodDelete}); err != nil { - t.Fatal(err) - } - // update time of 10.49.27.216 is more recently than 10.49.27.205 - if err := fipPlugin.ipam.AllocateSpecificIP(podKey.PoolPrefix(), net.ParseIP("10.49.27.216"), - floatingip.Attr{Policy: constant.ReleasePolicyPodDelete}); err != nil { - t.Fatal(err) - } - // check filter allocates recent ips for deployment pod from ip pool - if err := checkFilterCase(fipPlugin, filterCase{ - testPod: pod, expectFiltererd: []string{node3}, expectFailed: []string{drainedNode, nodeHasNoIP, node4}, - }, nodes); err != nil { - t.Fatal(err) - } - if err := checkIPKey(fipPlugin.ipam, "10.49.27.205", podKey.PoolPrefix()); err != nil { - t.Fatal(err) - } - if err := checkIPKey(fipPlugin.ipam, "10.49.27.216", podKey.KeyInDB); err != nil { - t.Fatal(err) - } -} - func checkIPKey(ipam floatingip.IPAM, checkIP, expectKey string) error { ip := net.ParseIP(checkIP) if ip == nil { @@ -762,56 +272,3 @@ func checkIPKey(ipam floatingip.IPAM, checkIP, expectKey string) error { } return nil } - -func checkBind(fipPlugin *FloatingIPPlugin, pod *corev1.Pod, nodeName, checkKey string, - expectSubnet *net.IPNet) (*floatingip.FloatingIPInfo, error) { - if err := fipPlugin.Bind(&schedulerapi.ExtenderBindingArgs{ - PodName: pod.Name, - PodNamespace: pod.Namespace, - Node: nodeName, - }); err != nil { - return nil, err - } - fipInfo, err := fipPlugin.ipam.First(checkKey) - if err != nil { - return nil, err - } - if fipInfo == nil { - return nil, fmt.Errorf("got nil ipInfo") - } - if !expectSubnet.Contains(fipInfo.IPInfo.IP.IP) { - return nil, fmt.Errorf("allocated ip %s is not in expect subnet %s", fipInfo.IPInfo.IP.IP.String(), - expectSubnet.String()) - } - return fipInfo, nil -} - -func TestReleaseIPOfFinishedPod(t *testing.T) { - for i, testCase := range []struct { - updatePodStatus func(pod *corev1.Pod) - }{ - {updatePodStatus: toFailedPod}, - {updatePodStatus: toSuccessPod}, - } { - pod := CreateStatefulSetPod("pod1-0", "ns1", nil) - podKey, _ := schedulerplugin_util.FormatKey(pod) - func() { - fipPlugin, stopChan, _ := createPluginTestNodes(t, pod) - fipPlugin.Run(stopChan) - defer func() { stopChan <- struct{}{} }() - fipInfo, err := checkBind(fipPlugin, pod, node3, podKey.KeyInDB, node3Subnet) - if err != nil { - t.Fatalf("case %d: %v", i, err) - } - testCase.updatePodStatus(pod) - if _, err := fipPlugin.Client.CoreV1().Pods(pod.Namespace).UpdateStatus(pod); err != nil { - t.Fatalf("case %d: %v", i, err) - } - if err := wait.Poll(time.Microsecond*10, time.Second*30, func() (done bool, err error) { - return checkIPKey(fipPlugin.ipam, fipInfo.FIP.IP.String(), "") == nil, nil - }); err != nil { - t.Fatalf("case %d: %v", i, err) - } - }() - } -} diff --git a/pkg/ipam/schedulerplugin/testing/util.go b/pkg/ipam/schedulerplugin/testing/util.go index 62e41a28..3a444270 100644 --- a/pkg/ipam/schedulerplugin/testing/util.go +++ b/pkg/ipam/schedulerplugin/testing/util.go @@ -19,6 +19,7 @@ package testing import ( "strings" + appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -76,6 +77,7 @@ func CreateTAppPod(name, namespace string, annotations map[string]string) *corev return pod } +// CreateSimplePod creates a pod given name, namespace and annotations for testing func CreateSimplePod(name, namespace string, annotations map[string]string) *corev1.Pod { pod := CreateStatefulSetPod(name+"-0", namespace, annotations) pod.Name = name @@ -83,8 +85,58 @@ func CreateSimplePod(name, namespace string, annotations map[string]string) *cor return pod } +// CreatePodWithKind creates a pod given name, namespace, owner kind and annotations for testing func CreatePodWithKind(name, namespace, kind string, annotations map[string]string) *corev1.Pod { pod := CreateStatefulSetPod(name, namespace, annotations) pod.OwnerReferences[0].Kind = kind return pod } + +// CreateDeployment creates a controller deployment for the given pod for testing +func CreateDeployment(podMeta v1.ObjectMeta, replicas int32) *appv1.Deployment { + parts := strings.Split(podMeta.OwnerReferences[0].Name, "-") + return &appv1.Deployment{ + ObjectMeta: v1.ObjectMeta{ + Name: strings.Join(parts[:len(parts)-1], "-"), + Namespace: podMeta.Namespace, + Labels: podMeta.Labels, + }, + Spec: appv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: podMeta, + }, + Replicas: &replicas, + Selector: &v1.LabelSelector{ + MatchLabels: podMeta.GetLabels(), + }, + }, + } +} + +// CreateStatefulSet creates a controller statefulset for the given pod for testing +func CreateStatefulSet(podMeta v1.ObjectMeta, replicas int32) *appv1.StatefulSet { + return &appv1.StatefulSet{ + ObjectMeta: v1.ObjectMeta{ + Name: podMeta.OwnerReferences[0].Name, + Namespace: podMeta.GetNamespace(), + Labels: podMeta.GetLabels(), + }, + Spec: appv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: podMeta, + }, + Replicas: &replicas, + Selector: &v1.LabelSelector{ + MatchLabels: podMeta.GetLabels(), + }, + }, + } +} + +// CreateNode creates a node for testing +func CreateNode(name string, labels map[string]string, address string) corev1.Node { + return corev1.Node{ + ObjectMeta: v1.ObjectMeta{Name: name, Labels: labels}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: address}}}, + } +} From adaf109f3c707f2481270cc7b0a29dca70795ca6 Mon Sep 17 00:00:00 2001 From: Chun Chen Date: Tue, 20 Oct 2020 18:06:36 +0800 Subject: [PATCH 3/3] Add tests for RequestIPRange - Fix if there are both unallocated and allocated ips in request ip ranges - Improve NodeSubnetsByIPRanges by caching pool in FloatingIP - fix updating workload from requesting multiple ips to only one ip wrongly allocates multiple ips - fix resync all pod ips - fix reserveIP --- pkg/api/galaxy/constant/constant.go | 2 - pkg/galaxy/server.go | 20 +- pkg/galaxy/server_test.go | 39 +++ pkg/ipam/api/api.go | 52 ++-- pkg/ipam/api/pool.go | 2 +- .../testing/fake_cloud_provider.go | 29 +- pkg/ipam/floatingip/floatingip.go | 16 +- pkg/ipam/floatingip/ipam.go | 15 +- pkg/ipam/floatingip/ipam_crd.go | 214 ++++++--------- pkg/ipam/floatingip/ipam_crd_test.go | 249 +++++++++++++++--- pkg/ipam/floatingip/store_crd.go | 4 +- pkg/ipam/floatingip/store_crd_test.go | 4 +- pkg/ipam/schedulerplugin/bind.go | 63 +++-- pkg/ipam/schedulerplugin/bind_test.go | 219 ++++++++++++--- pkg/ipam/schedulerplugin/deployment_test.go | 2 +- pkg/ipam/schedulerplugin/filter.go | 45 +++- pkg/ipam/schedulerplugin/filter_test.go | 88 ++++++- .../schedulerplugin/floatingip_plugin_test.go | 5 + pkg/ipam/schedulerplugin/ipam.go | 8 +- pkg/ipam/schedulerplugin/resync.go | 11 +- pkg/ipam/schedulerplugin/statefulset_test.go | 2 +- pkg/utils/nets/ip.go | 16 ++ 22 files changed, 791 insertions(+), 314 deletions(-) create mode 100644 pkg/galaxy/server_test.go diff --git a/pkg/api/galaxy/constant/constant.go b/pkg/api/galaxy/constant/constant.go index 697f406b..d5f85899 100644 --- a/pkg/api/galaxy/constant/constant.go +++ b/pkg/api/galaxy/constant/constant.go @@ -30,8 +30,6 @@ const ( MultusCNIAnnotation = "k8s.v1.cni.cncf.io/networks" - CommonCNIArgsKey = "common" - // For fip crd object which has this label, it's reserved by admin manually. IPAM will not allocate it to pods. ReserveFIPLabel = "reserved" diff --git a/pkg/galaxy/server.go b/pkg/galaxy/server.go index c57512a6..614f824a 100644 --- a/pkg/galaxy/server.go +++ b/pkg/galaxy/server.go @@ -217,11 +217,9 @@ func (g *Galaxy) resolveNetworks(req *galaxyapi.PodRequest, pod *corev1.Pod) ([] if err != nil { return nil, err } - if commonArgs, exist := extendedCNIArgs[constant.CommonCNIArgsKey]; exist { - for i := range networkInfos { - for k, v := range commonArgs { - networkInfos[i].Args[k] = string([]byte(v)) - } + for i := range networkInfos { + for k, v := range extendedCNIArgs { + networkInfos[i].Args[k] = string(v) } } glog.V(4).Infof("pod %s_%s networkInfo %v", pod.Name, pod.Namespace, networkInfos) @@ -263,7 +261,7 @@ func (g *Galaxy) cmdAdd(req *galaxyapi.PodRequest, pod *corev1.Pod) (types.Resul } // parseExtendedCNIArgs parses extended cni args from pod's annotation -func parseExtendedCNIArgs(pod *corev1.Pod) (map[string]map[string]json.RawMessage, error) { +func parseExtendedCNIArgs(pod *corev1.Pod) (map[string]json.RawMessage, error) { if pod.Annotations == nil { return nil, nil } @@ -271,11 +269,15 @@ func parseExtendedCNIArgs(pod *corev1.Pod) (map[string]map[string]json.RawMessag if args == "" { return nil, nil } - argsMap := map[string]map[string]json.RawMessage{} - if err := json.Unmarshal([]byte(args), &argsMap); err != nil { + // CniArgs is the cni args in pod annotation + var cniArgs struct { + // Common is the common args for cni plugins to setup network + Common map[string]json.RawMessage `json:"common"` + } + if err := json.Unmarshal([]byte(args), &cniArgs); err != nil { return nil, fmt.Errorf("failed to unmarshal cni args %s: %v", args, err) } - return argsMap, nil + return cniArgs.Common, nil } func (g *Galaxy) setupIPtables() error { diff --git a/pkg/galaxy/server_test.go b/pkg/galaxy/server_test.go new file mode 100644 index 00000000..645d1c15 --- /dev/null +++ b/pkg/galaxy/server_test.go @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package galaxy + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "tkestack.io/galaxy/pkg/api/galaxy/constant" +) + +func TestParseExtendedCNIArgs(t *testing.T) { + m, err := parseExtendedCNIArgs(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + constant.ExtendedCNIArgsAnnotation: `{"request_ip_range":[["10.0.0.2~10.0.0.30"],["10.0.0.200~10.0.0.238"]],"common":{"ipinfos":[{"ip":"10.0.0.3/24","vlan":0,"gateway":"10.0.0.1"},{"ip":"10.0.0.200/24","vlan":0,"gateway":"10.0.0.1"}]}}`, + }}}) + if err != nil { + t.Fatal(err) + } + if val, ok := m["ipinfos"]; !ok { + t.Fatal() + } else if string(val) != `[{"ip":"10.0.0.3/24","vlan":0,"gateway":"10.0.0.1"},{"ip":"10.0.0.200/24","vlan":0,"gateway":"10.0.0.1"}]` { + t.Fatal() + } +} diff --git a/pkg/ipam/api/api.go b/pkg/ipam/api/api.go index aec54c6c..661e3d23 100644 --- a/pkg/ipam/api/api.go +++ b/pkg/ipam/api/api.go @@ -322,35 +322,39 @@ func (c *Controller) ReleaseIPs(req *restful.Request, resp *restful.Response) { // listIPs lists ips from ipams func listIPs(keyword string, ipam floatingip.IPAM, fuzzyQuery bool) ([]FloatingIP, error) { - var fips []floatingip.FloatingIP - var err error + var result []FloatingIP if fuzzyQuery { - fips, err = ipam.ByKeyword(keyword) + fips, err := ipam.ByKeyword(keyword) + if err != nil { + return nil, err + } + for i := range fips { + result = append(result, convert(&fips[i])) + } } else { - fips, err = ipam.ByPrefix(keyword) - } - if err != nil { - return nil, err + fips, err := ipam.ByPrefix(keyword) + if err != nil { + return nil, err + } + for i := range fips { + result = append(result, convert(&fips[i].FloatingIP)) + } } - return transform(fips), nil + return result, nil } -// transform converts `floatingip.FloatingIP` slice to `FloatingIP` slice -func transform(fips []floatingip.FloatingIP) []FloatingIP { - var res []FloatingIP - for i := range fips { - keyObj := util.ParseKey(fips[i].Key) - res = append(res, FloatingIP{IP: fips[i].IP.String(), - Namespace: keyObj.Namespace, - AppName: keyObj.AppName, - PodName: keyObj.PodName, - PoolName: keyObj.PoolName, - AppType: toAppType(keyObj.AppTypePrefix), - Policy: fips[i].Policy, - UpdateTime: fips[i].UpdatedAt, - labels: fips[i].Labels}) - } - return res +// convert converts `floatingip.FloatingIP` to `FloatingIP` +func convert(fip *floatingip.FloatingIP) FloatingIP { + keyObj := util.ParseKey(fip.Key) + return FloatingIP{IP: fip.IP.String(), + Namespace: keyObj.Namespace, + AppName: keyObj.AppName, + PodName: keyObj.PodName, + PoolName: keyObj.PoolName, + AppType: toAppType(keyObj.AppTypePrefix), + Policy: fip.Policy, + UpdateTime: fip.UpdatedAt, + labels: fip.Labels} } // batchReleaseIPs release ips from ipams diff --git a/pkg/ipam/api/pool.go b/pkg/ipam/api/pool.go index a02693fb..6c341037 100644 --- a/pkg/ipam/api/pool.go +++ b/pkg/ipam/api/pool.go @@ -144,7 +144,7 @@ func (c *PoolController) preAllocateIP(req *restful.Request, resp *restful.Respo httputil.InternalError(resp, err) return } - subnetSet, err := c.IPAM.NodeSubnetsByKeyAndIPRanges("", nil) + subnetSet, err := c.IPAM.NodeSubnetsByIPRanges(nil) if err != nil { httputil.InternalError(resp, err) return diff --git a/pkg/ipam/cloudprovider/testing/fake_cloud_provider.go b/pkg/ipam/cloudprovider/testing/fake_cloud_provider.go index 4affd759..1ba5eae7 100644 --- a/pkg/ipam/cloudprovider/testing/fake_cloud_provider.go +++ b/pkg/ipam/cloudprovider/testing/fake_cloud_provider.go @@ -19,41 +19,34 @@ package testing import ( "fmt" + "tkestack.io/galaxy/pkg/ipam/cloudprovider" "tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc" ) +var _ cloudprovider.CloudProvider = &FakeCloudProvider{} + // FakeCloudProvider is a fake cloud provider for testing type FakeCloudProvider struct { - ExpectIP string - ExpectNode string - InvokedAssignIP bool - InvokedUnAssignIP bool + Assigned map[string]string // Assigned ipaddress to nodeName + UnAssigned map[string]string // UnAssigned ipaddress to nodeName +} + +func NewFakeCloudProvider() *FakeCloudProvider { + return &FakeCloudProvider{Assigned: map[string]string{}, UnAssigned: map[string]string{}} } func (f *FakeCloudProvider) AssignIP(in *rpc.AssignIPRequest) (*rpc.AssignIPReply, error) { - f.InvokedAssignIP = true if in == nil { return nil, fmt.Errorf("nil request") } - if in.IPAddress != f.ExpectIP { - return nil, fmt.Errorf("expect ip %s, got %s", f.ExpectIP, in.IPAddress) - } - if in.NodeName != f.ExpectNode { - return nil, fmt.Errorf("expect node name %s, got %s", f.ExpectNode, in.NodeName) - } + f.Assigned[in.IPAddress] = in.NodeName return &rpc.AssignIPReply{Success: true}, nil } func (f *FakeCloudProvider) UnAssignIP(in *rpc.UnAssignIPRequest) (*rpc.UnAssignIPReply, error) { - f.InvokedUnAssignIP = true if in == nil { return nil, fmt.Errorf("nil request") } - if in.IPAddress != f.ExpectIP { - return nil, fmt.Errorf("expect ip %s, got %s", f.ExpectIP, in.IPAddress) - } - if in.NodeName != f.ExpectNode { - return nil, fmt.Errorf("expect node name %s, got %s", f.ExpectNode, in.NodeName) - } + f.UnAssigned[in.IPAddress] = in.NodeName return &rpc.UnAssignIPReply{Success: true}, nil } diff --git a/pkg/ipam/floatingip/floatingip.go b/pkg/ipam/floatingip/floatingip.go index 3c97cf2b..4b89a381 100644 --- a/pkg/ipam/floatingip/floatingip.go +++ b/pkg/ipam/floatingip/floatingip.go @@ -31,23 +31,23 @@ import ( // FloatingIP defines a floating ip type FloatingIP struct { Key string - Subnets sets.String // node subnet, not container ip's subnet IP net.IP UpdatedAt time.Time Labels map[string]string Policy uint16 NodeName string PodUid string + pool *FloatingIPPool } func (f FloatingIP) String() string { - return fmt.Sprintf("FloatingIP{ip:%s key:%s policy:%d nodeName:%s podUid:%s subnets:%v}", - f.IP.String(), f.Key, f.Policy, f.NodeName, f.PodUid, f.Subnets) + return fmt.Sprintf("FloatingIP{ip:%s key:%s policy:%d nodeName:%s podUid:%s}", + f.IP.String(), f.Key, f.Policy, f.NodeName, f.PodUid) } // New creates a new FloatingIP -func New(ip net.IP, subnets sets.String, key string, attr *Attr, updateAt time.Time) *FloatingIP { - fip := &FloatingIP{IP: ip, Subnets: subnets} +func New(pool *FloatingIPPool, ip net.IP, key string, attr *Attr, updateAt time.Time) *FloatingIP { + fip := &FloatingIP{IP: ip, pool: pool} fip.Assign(key, attr, updateAt) return fip } @@ -65,8 +65,8 @@ func (f *FloatingIP) Assign(key string, attr *Attr, updateAt time.Time) *Floatin // CloneWith creates a new FloatingIP and updates key, attr, updatedAt func (f *FloatingIP) CloneWith(key string, attr *Attr, updateAt time.Time) *FloatingIP { fip := &FloatingIP{ - IP: f.IP, - Subnets: f.Subnets, + IP: f.IP, + pool: f.pool, } return fip.Assign(key, attr, updateAt) } @@ -76,6 +76,8 @@ type FloatingIPPool struct { NodeSubnets []*net.IPNet // the node subnets nets.SparseSubnet sync.RWMutex + nodeSubnets sets.String // the node subnets, string set format + index int // the index of []FloatingIPPool } // FloatingIPPoolConf is FloatingIP config structure. diff --git a/pkg/ipam/floatingip/ipam.go b/pkg/ipam/floatingip/ipam.go index 504bd9d3..adc77c85 100644 --- a/pkg/ipam/floatingip/ipam.go +++ b/pkg/ipam/floatingip/ipam.go @@ -44,6 +44,7 @@ type IPAM interface { // AllocateInSubnet allocate subnet of IPs. AllocateInSubnet(string, *net.IPNet, Attr) (net.IP, error) // AllocateInSubnetsAndIPRange allocates an ip for each ip range array of the input node subnet. + // It guarantees allocating all ips or no ips. AllocateInSubnetsAndIPRange(string, *net.IPNet, [][]nets.IPRange, Attr) ([]net.IP, error) // AllocateInSubnetWithKey allocate a floatingIP in given subnet and key. AllocateInSubnetWithKey(oldK, newK, subnet string, attr Attr) error @@ -59,16 +60,19 @@ type IPAM interface { // ByIP transform a given IP to FloatingIP struct. ByIP(net.IP) (FloatingIP, error) // ByPrefix filter floatingIPs by prefix key. - ByPrefix(string) ([]FloatingIP, error) + ByPrefix(string) ([]*FloatingIPInfo, error) // ByKeyword returns floatingIP set by a given keyword. ByKeyword(string) ([]FloatingIP, error) - // ByKeyAndIPRanges finds an ip for each iprange array by key, and returns all fips + // ByKeyAndIPRanges finds an allocated ip for each []nets.IPRange by key. + // If input [][]nets.IPRange is nil or empty, it finds all allocated ips by key. + // Otherwise, it always return the same size of []*FloatingIPInfo as [][]nets.IPRange, the element of + // []*FloatingIPInfo may be nil when it can't find an allocated ip for the same index of []iprange. ByKeyAndIPRanges(string, [][]nets.IPRange) ([]*FloatingIPInfo, error) // NodeSubnets returns node's subnet. NodeSubnet(net.IP) *net.IPNet - // NodeSubnetsByKeyAndIPRanges finds an ip for each iprange array by key, and returns their intersection + // NodeSubnetsByIPRanges finds an unallocated ip for each []nets.IPRange, and returns their intersection // node subnets. - NodeSubnetsByKeyAndIPRanges(key string, ipranges [][]nets.IPRange) (sets.String, error) + NodeSubnetsByIPRanges(ipranges [][]nets.IPRange) (sets.String, error) // Name returns IPAM's name. Name() string // implements metrics Collector interface @@ -78,5 +82,6 @@ type IPAM interface { // FloatingIPInfo is floatingIP information type FloatingIPInfo struct { IPInfo constant.IPInfo - FIP FloatingIP + FloatingIP + NodeSubnets sets.String } diff --git a/pkg/ipam/floatingip/ipam_crd.go b/pkg/ipam/floatingip/ipam_crd.go index 792693db..d5712c61 100644 --- a/pkg/ipam/floatingip/ipam_crd.go +++ b/pkg/ipam/floatingip/ipam_crd.go @@ -105,7 +105,7 @@ func (ci *crdIpam) AllocateSpecificIP(key string, ip net.IP, attr Attr) error { if !find { return fmt.Errorf("failed to find floating ip by %s in cache", ipStr) } - allocated := New(ip, spec.Subnets, key, &attr, time.Now()) + allocated := New(spec.pool, ip, key, &attr, time.Now()) if err := ci.createFloatingIP(allocated); err != nil { glog.Errorf("failed to create floatingIP %s: %v", ipStr, err) return err @@ -128,10 +128,10 @@ func (ci *crdIpam) AllocateInSubnet(key string, nodeSubnet *net.IPNet, attr Attr nodeSubnetStr := nodeSubnet.String() for k, v := range ci.unallocatedFIPs { //find an unallocated fip, then use it - if v.Subnets.Has(nodeSubnetStr) { + if v.pool.nodeSubnets.Has(nodeSubnetStr) { ipStr = k // we never updates ip or subnet object, it's ok to share these objs. - allocated := New(v.IP, v.Subnets, key, &attr, time.Now()) + allocated := New(v.pool, v.IP, key, &attr, time.Now()) if err := ci.createFloatingIP(allocated); err != nil { glog.Errorf("failed to create floatingIP %s: %v", ipStr, err) return nil, err @@ -157,7 +157,7 @@ func (ci *crdIpam) AllocateInSubnetWithKey(oldK, newK, subnet string, attr Attr) ) //find latest floatingIP by updateTime. for _, v := range ci.allocatedFIPs { - if v.Key == oldK && v.Subnets.Has(subnet) { + if v.Key == oldK && v.pool.nodeSubnets.Has(subnet) { if v.UpdatedAt.UnixNano() > recordTs { latest = v recordTs = v.UpdatedAt.UnixNano() @@ -187,7 +187,7 @@ func (ci *crdIpam) ReserveIP(oldK, newK string, attr Attr) (bool, error) { if v.Key == oldK { if oldK == newK && v.PodUid == attr.Uid && v.NodeName == attr.NodeName { // nothing changed - return false, nil + continue } attr.Policy = constant.ReleasePolicy(v.Policy) if err := ci.updateFloatingIP(v.CloneWith(newK, &attr, date)); err != nil { @@ -198,10 +198,7 @@ func (ci *crdIpam) ReserveIP(oldK, newK string, attr Attr) (bool, error) { reserved = true } } - if reserved { - return true, nil - } - return false, fmt.Errorf("failed to find floatIP by key %s", oldK) + return reserved, nil } // UpdateAttr update floatingIP's release policy and attr according to ip and key @@ -248,32 +245,12 @@ func (ci *crdIpam) Release(key string, ip net.IP) error { func (ci *crdIpam) First(key string) (*FloatingIPInfo, error) { ci.cacheLock.RLock() defer ci.cacheLock.RUnlock() - var fip *FloatingIP for _, spec := range ci.allocatedFIPs { if spec.Key == key { - fip = spec + return ci.toFloatingIPInfo(spec), nil } } - if fip == nil { - return nil, nil - } - for _, fips := range ci.FloatingIPs { - if fips.Contains(fip.IP) { - ip := nets.IPNet(net.IPNet{ - IP: fip.IP, - Mask: fips.Mask, - }) - return &FloatingIPInfo{ - IPInfo: constant.IPInfo{ - IP: &ip, - Vlan: fips.Vlan, - Gateway: fips.Gateway, - }, - FIP: *fip, - }, nil - } - } - return nil, fmt.Errorf("could not find match floating ip config for ip %s", fip.IP.String()) + return nil, nil } // ByIP transform a given IP to FloatingIP struct. @@ -292,18 +269,18 @@ func (ci *crdIpam) ByIP(ip net.IP) (FloatingIP, error) { } // ByPrefix filter floatingIPs by prefix key. -func (ci *crdIpam) ByPrefix(prefix string) ([]FloatingIP, error) { - var fips []FloatingIP +func (ci *crdIpam) ByPrefix(prefix string) ([]*FloatingIPInfo, error) { + var fips []*FloatingIPInfo ci.cacheLock.RLock() defer ci.cacheLock.RUnlock() for _, spec := range ci.allocatedFIPs { if strings.HasPrefix(spec.Key, prefix) { - fips = append(fips, *spec) + fips = append(fips, ci.toFloatingIPInfo(spec)) } } if prefix == "" { for _, spec := range ci.unallocatedFIPs { - fips = append(fips, *spec) + fips = append(fips, ci.toFloatingIPInfo(spec)) } } return fips, nil @@ -323,51 +300,52 @@ func (ci *crdIpam) NodeSubnet(nodeIP net.IP) *net.IPNet { return nil } -func (ci *crdIpam) NodeSubnetsByKeyAndIPRanges(key string, ipranges [][]nets.IPRange) (sets.String, error) { +func (ci *crdIpam) NodeSubnetsByIPRanges(ipranges [][]nets.IPRange) (sets.String, error) { subnetSet := sets.NewString() + insertSubnet := func(poolIndexSet sets.Int, subnetSet sets.String) { + for _, index := range poolIndexSet.UnsortedList() { + if index >= 0 && index <= len(ci.FloatingIPs)-1 { + subnetSet.Insert(ci.FloatingIPs[index].nodeSubnets.UnsortedList()...) + } + } + } ci.cacheLock.RLock() defer ci.cacheLock.RUnlock() if len(ipranges) == 0 { - if key == "" { - for _, val := range ci.unallocatedFIPs { - subnetSet.Insert(val.Subnets.UnsortedList()...) - } - } else { - // key is not be empty - for _, spec := range ci.allocatedFIPs { - if spec.Key == key { - subnetSet.Insert(spec.Subnets.UnsortedList()...) - } - } + poolIndexSet := sets.NewInt() + for _, val := range ci.unallocatedFIPs { + poolIndexSet.Insert(val.pool.index) } + insertSubnet(poolIndexSet, subnetSet) return subnetSet, nil } - for i, ranges := range ipranges { - partSubnets := sets.NewString() + for _, ranges := range ipranges { + poolIndexSet := sets.NewInt() walkIPRanges(ranges, func(ip net.IP) bool { ipStr := ip.String() - if key == "" { - if fip, ok := ci.unallocatedFIPs[ipStr]; !ok { - return false - } else { - partSubnets.Insert(fip.Subnets.UnsortedList()...) - } + if fip, ok := ci.unallocatedFIPs[ipStr]; !ok { + return false } else { - // TODO if there is an allocated ip in iprange1 and no allocated ip in iprange2 - if fip, ok := ci.allocatedFIPs[ipStr]; !ok || fip.Key != key { - return false - } else { - partSubnets.Insert(fip.Subnets.UnsortedList()...) - } + poolIndexSet.Insert(fip.pool.index) } return false }) - if i == 0 { - subnetSet = partSubnets + // no ip left in this []nets.IPRange + if poolIndexSet.Len() == 0 { + glog.V(3).Infof("no enough ips for ip range %v", ranges) + return sets.NewString(), nil + } + if subnetSet.Len() == 0 { + insertSubnet(poolIndexSet, subnetSet) } else { - subnetSet = subnetSet.Intersection(partSubnets) + partset := sets.NewString() + insertSubnet(poolIndexSet, partset) + subnetSet = subnetSet.Intersection(partset) } } + // TODO try to allocate to check if each subnet has enough ips when [][]nets.IPRange has overlap ranges + // e.g. if [][]nets.IPRange = [["10.0.0.1","10.0.1.1~10.0.1.3"]["10.0.0.1","10.0.1.1~10.0.1.3"]], we should + // return 10.0.1.0/24 and exclude 10.0.0.0/24 return subnetSet, nil } @@ -399,13 +377,13 @@ func (ci *crdIpam) ConfigurePool(floatIPs []*FloatingIPPool) error { return err } glog.V(3).Infof("floating ip config %v", floatIPs) - nodeSubnets := make([]sets.String, len(floatIPs)) - for i, fipConf := range floatIPs { + for index, fipConf := range floatIPs { subnetSet := sets.NewString() for i := range fipConf.NodeSubnets { subnetSet.Insert(fipConf.NodeSubnets[i].String()) } - nodeSubnets[i] = subnetSet + fipConf.nodeSubnets = subnetSet + fipConf.index = index } var deletingIPs []string tmpCacheAllocated := make(map[string]*FloatingIP) @@ -413,20 +391,11 @@ func (ci *crdIpam) ConfigurePool(floatIPs []*FloatingIPPool) error { for _, ip := range ips.Items { netIP := net.ParseIP(ip.Name) found := false - for i, fipConf := range floatIPs { + for _, fipConf := range floatIPs { if fipConf.IPNet().Contains(netIP) && fipConf.Contains(netIP) { found = true //ip in config, insert it into cache - tmpFip := &FloatingIP{ - IP: netIP, - Key: ip.Spec.Key, - Policy: uint16(ip.Spec.Policy), - // Since subnets may change and for reserved fips crds created by user manually, subnets may not be - // correct, assign it to the latest config instead of crd value - // TODO we can delete subnets field from crd? - Subnets: nodeSubnets[i], - UpdatedAt: ip.Spec.UpdateTime.Time, - } + tmpFip := New(fipConf, netIP, ip.Spec.Key, &Attr{Policy: ip.Spec.Policy}, ip.Spec.UpdateTime.Time) if err := tmpFip.unmarshalAttr(ip.Spec.Attribute); err != nil { glog.Error(err) } @@ -455,18 +424,11 @@ func (ci *crdIpam) ConfigurePool(floatIPs []*FloatingIPPool) error { now := time.Now() // fresh unallocated floatIP tmpCacheUnallocated := make(map[string]*FloatingIP) - for i, fipConf := range floatIPs { - subnetSet := nodeSubnets[i] + for _, fipConf := range floatIPs { walkIPRanges(fipConf.IPRanges, func(ip net.IP) bool { ipStr := ip.String() if _, contain := ci.allocatedFIPs[ipStr]; !contain { - tmpFip := &FloatingIP{ - IP: ip, - Key: "", - Policy: uint16(constant.ReleasePolicyPodDelete), - Subnets: subnetSet, - UpdatedAt: now, - } + tmpFip := New(fipConf, ip, "", &Attr{Policy: constant.ReleasePolicyPodDelete}, now) tmpCacheUnallocated[ipStr] = tmpFip } return false @@ -502,9 +464,6 @@ func (ci *crdIpam) ByKeyword(keyword string) ([]FloatingIP, error) { var fips []FloatingIP ci.cacheLock.RLock() defer ci.cacheLock.RUnlock() - if ci.allocatedFIPs == nil { - return fips, nil - } for _, spec := range ci.allocatedFIPs { if strings.Contains(spec.Key, keyword) { fips = append(fips, *spec) @@ -589,6 +548,8 @@ func (ci *crdIpam) Collect(ch chan<- prometheus.Metric) { } // AllocateInSubnetsAndIPRange allocates an ip for each ip range array of the input node subnet. +// It guarantees allocating all ips or no ips. +// TODO Fix allocation for [][]nets.IPRange [["10.0.0.1~10.0.0.2"]["10.0.0.1"]] func (ci *crdIpam) AllocateInSubnetsAndIPRange(key string, nodeSubnet *net.IPNet, ipranges [][]nets.IPRange, attr Attr) ([]net.IP, error) { if nodeSubnet == nil { @@ -606,19 +567,24 @@ func (ci *crdIpam) AllocateInSubnetsAndIPRange(key string, nodeSubnet *net.IPNet defer ci.cacheLock.Unlock() // pick ips to allocate, one per []nets.IPRange var allocatedIPStrs []string + // allocatedIPSet is the allocated ips in the previous loop, to avoid count in duplicate ips + allocatedIPSet := sets.NewString() for _, ranges := range ipranges { var allocated bool walkIPRanges(ranges, func(ip net.IP) bool { ipStr := ip.String() - if fip, ok := ci.unallocatedFIPs[ipStr]; !ok || !fip.Subnets.Has(nodeSubnet.String()) { + if fip, ok := ci.unallocatedFIPs[ipStr]; !ok || !fip.pool.nodeSubnets.Has(nodeSubnet.String()) || + allocatedIPSet.Has(ipStr) { return false } allocatedIPStrs = append(allocatedIPStrs, ipStr) + allocatedIPSet.Insert(ipStr) allocated = true return true }) if !allocated { - glog.Warningf("not enough ip to allocate for %s in %s, %v", key, nodeSubnet.String(), ipranges) + glog.V(3).Infof("no enough ips to allocate for %s node subnet %s, ip range %v", key, + nodeSubnet.String(), ipranges) return nil, ErrNoEnoughIP } } @@ -628,7 +594,7 @@ func (ci *crdIpam) AllocateInSubnetsAndIPRange(key string, nodeSubnet *net.IPNet for i, allocatedIPStr := range allocatedIPStrs { v := ci.unallocatedFIPs[allocatedIPStr] // we never updates ip or subnet object, it's ok to share these objs. - allocated := New(v.IP, v.Subnets, key, &attr, time.Now()) + allocated := New(v.pool, v.IP, key, &attr, time.Now()) if err := ci.createFloatingIP(allocated); err != nil { glog.Errorf("failed to create floatingIP %s: %v", allocatedIPStr, err) // rollback all allocated ips @@ -652,42 +618,31 @@ func (ci *crdIpam) AllocateInSubnetsAndIPRange(key string, nodeSubnet *net.IPNet return allocatedIPs, nil } -// ByKeyAndIPRanges finds an ip for each iprange array by key, and returns all fips +// ByKeyAndIPRanges finds an allocated ip for each []iprange by key. +// If input [][]nets.IPRange is nil or empty, it finds all allocated ips by key. +// Otherwise, it always return the same size of []*FloatingIPInfo as [][]nets.IPRange, the element of +// []*FloatingIPInfo may be nil when it can't find an allocated ip for the same index of []iprange. func (ci *crdIpam) ByKeyAndIPRanges(key string, ipranges [][]nets.IPRange) ([]*FloatingIPInfo, error) { ci.cacheLock.RLock() defer ci.cacheLock.RUnlock() var ipinfos []*FloatingIPInfo if len(ipranges) != 0 { - for _, ranges := range ipranges { - var err error + ipinfos = make([]*FloatingIPInfo, len(ipranges)) + for i, ranges := range ipranges { walkIPRanges(ranges, func(ip net.IP) bool { ipStr := ip.String() - fip := ci.allocatedFIPs[ipStr] - if fip.Key != key { + fip, ok := ci.allocatedFIPs[ipStr] + if !ok || fip.Key != key { return false } - fipInfo := ci.toFloatingIPInfo(fip) - if fipInfo == nil { - err = fmt.Errorf("could not find match floating ip config for ip %s", fip.IP.String()) - } else { - ipinfos = append(ipinfos, fipInfo) - } + ipinfos[i] = ci.toFloatingIPInfo(fip) return true }) - if err != nil { - return nil, err - } } } else { for _, fip := range ci.allocatedFIPs { - if fip.Key != key { - continue - } - fipInfo := ci.toFloatingIPInfo(fip) - if fipInfo == nil { - return nil, fmt.Errorf("could not find match floating ip config for ip %s", fip.IP.String()) - } else { - ipinfos = append(ipinfos, fipInfo) + if fip.Key == key { + ipinfos = append(ipinfos, ci.toFloatingIPInfo(fip)) } } } @@ -695,23 +650,20 @@ func (ci *crdIpam) ByKeyAndIPRanges(key string, ipranges [][]nets.IPRange) ([]*F } func (ci *crdIpam) toFloatingIPInfo(fip *FloatingIP) *FloatingIPInfo { - for _, fipPool := range ci.FloatingIPs { - if fipPool.Contains(fip.IP) { - ip := nets.IPNet(net.IPNet{ - IP: fip.IP, - Mask: fipPool.Mask, - }) - return &FloatingIPInfo{ - IPInfo: constant.IPInfo{ - IP: &ip, - Vlan: fipPool.Vlan, - Gateway: fipPool.Gateway, - }, - FIP: *fip, - } - } + fipPool := fip.pool + ip := nets.IPNet(net.IPNet{ + IP: fip.IP, + Mask: fipPool.Mask, + }) + return &FloatingIPInfo{ + IPInfo: constant.IPInfo{ + IP: &ip, + Vlan: fipPool.Vlan, + Gateway: fipPool.Gateway, + }, + FloatingIP: *fip, + NodeSubnets: sets.NewString(fipPool.nodeSubnets.UnsortedList()...), } - return nil } // walkIPRanges walks all ips in the ranges, and calls f for each ip. If f returns true, walkIPRanges stops. diff --git a/pkg/ipam/floatingip/ipam_crd_test.go b/pkg/ipam/floatingip/ipam_crd_test.go index 4c5ac73a..3e4fa658 100644 --- a/pkg/ipam/floatingip/ipam_crd_test.go +++ b/pkg/ipam/floatingip/ipam_crd_test.go @@ -21,7 +21,6 @@ import ( "fmt" "net" "reflect" - "strings" "testing" "time" @@ -32,6 +31,7 @@ import ( fakeGalaxyCli "tkestack.io/galaxy/pkg/ipam/client/clientset/versioned/fake" crdInformer "tkestack.io/galaxy/pkg/ipam/client/informers/externalversions" "tkestack.io/galaxy/pkg/ipam/utils" + "tkestack.io/galaxy/pkg/utils/nets" ) const ( @@ -61,6 +61,7 @@ var ( node6FIPSubnet = &net.IPNet{IP: net.ParseIP("10.0.80.0"), Mask: mask24} node7IPNet = node6IPNet1 node7FIPSubnet = &net.IPNet{IP: net.ParseIP("10.0.81.0"), Mask: mask24} + allNodeSubnet = []*net.IPNet{node1IPNet, node2IPNet, node3IPNet, node4IPNet, node5IPNet1, node5IPNet2, node6IPNet1, node6IPNet2, node7IPNet} ) func createIPAM(t *testing.T, objs ...runtime.Object) (*crdIpam, crdInformer.SharedInformerFactory) { @@ -110,7 +111,6 @@ func TestConfigurePoolWithAllocatedIP(t *testing.T) { expectFip := &FloatingIP{ IP: net.ParseIP("10.49.27.205"), Key: "pod2", - Subnets: sets.NewString("subnet1"), // assign a bad subnet to test if it can be correct Policy: 0, UpdatedAt: time.Now(), } @@ -133,14 +133,9 @@ func TestConfigurePoolWithAllocatedIP(t *testing.T) { if fip.Key != expectFip.Key { t.Fatal() } - subnetsStr := strings.Join(fip.Subnets.List(), ",") - // test subnets is equal the lastest configure value instead of the stored value in crd - if subnetsStr != node1IPNet.String() { - t.Fatal(subnetsStr) - } } -func TestCRDAllocateSpecificIP(t *testing.T) { +func TestAllocateSpecificIP(t *testing.T) { now := time.Now() ipam := createTestCrdIPAM(t) if err := ipam.AllocateSpecificIP("pod1", net.ParseIP("10.49.27.205"), @@ -157,7 +152,7 @@ func TestCRDAllocateSpecificIP(t *testing.T) { if !allocated.UpdatedAt.After(now) { t.Fatal(allocated.UpdatedAt) } - if `FloatingIP{ip:10.49.27.205 key:pod1 policy:2 nodeName:212 podUid:xx1 subnets:map[10.49.27.0/24:{}]}` != + if `FloatingIP{ip:10.49.27.205 key:pod1 policy:2 nodeName:212 podUid:xx1}` != fmt.Sprintf("%+v", allocated) { t.Fatal(fmt.Sprintf("%+v", allocated)) } @@ -166,31 +161,34 @@ func TestCRDAllocateSpecificIP(t *testing.T) { } } -func checkFIP(ipam *crdIpam, expect string) error { +func checkFIP(ipam *crdIpam, expect ...string) error { fips, err := ipam.client.GalaxyV1alpha1().FloatingIPs().List(v1.ListOptions{}) if err != nil { return err } - if len(fips.Items) != 1 { - return fmt.Errorf("expect 1 fip, found %v", fips) + if len(fips.Items) != len(expect) { + return fmt.Errorf("expect %d fip, found %v", len(expect), fips) } - fip := fips.Items[0] - fip.Spec.UpdateTime = v1.Time{time.Time{}} - data, err := json.Marshal(fip) - if err != nil { - return err - } - if expect != string(data) { - return fmt.Errorf("expect %s, found %s", expect, string(data)) + for i, fip := range fips.Items { + fip.Spec.UpdateTime = v1.Time{time.Time{}} + data, err := json.Marshal(fip) + if err != nil { + return err + } + if expect[i] != string(data) { + return fmt.Errorf("case %d, expect crd %s, found %s", i, expect[i], string(data)) + } } return nil } -func TestCRDReserveIP(t *testing.T) { +func TestReserveIP(t *testing.T) { ipam := createTestCrdIPAM(t) - if err := ipam.AllocateSpecificIP("pod1", net.ParseIP("10.49.27.205"), - Attr{Policy: constant.ReleasePolicyNever, NodeName: "node1", Uid: "xx1"}); err != nil { - t.Fatal(err) + for _, ip := range []string{"10.49.27.205", "10.49.27.216"} { + if err := ipam.AllocateSpecificIP("pod1", net.ParseIP(ip), + Attr{Policy: constant.ReleasePolicyNever, NodeName: "node1", Uid: "xx1"}); err != nil { + t.Fatal(err) + } } newAttr := Attr{NodeName: "node2", Uid: "xx2", Policy: constant.ReleasePolicyNever} if reserved, err := ipam.ReserveIP("pod1", "p1", newAttr); err != nil { @@ -198,10 +196,14 @@ func TestCRDReserveIP(t *testing.T) { } else if !reserved { t.Fatal() } - if err := checkIPKeyAttr(ipam, "10.49.27.205", "p1", &newAttr); err != nil { - t.Fatal(err) + for _, ip := range []string{"10.49.27.205", "10.49.27.216"} { + if err := checkIPKeyAttr(ipam, ip, "p1", &newAttr); err != nil { + t.Fatal(err) + } } - if err := checkFIP(ipam, `{"kind":"FloatingIP","apiVersion":"galaxy.k8s.io/v1alpha1","metadata":{"name":"10.49.27.205","creationTimestamp":null,"labels":{"ipType":"internalIP"}},"spec":{"key":"p1","attribute":"{\"NodeName\":\"node2\",\"Uid\":\"xx2\"}","policy":2,"subnet":"10.49.27.0/24","updateTime":null}}`); err != nil { + if err := checkFIP(ipam, + `{"kind":"FloatingIP","apiVersion":"galaxy.k8s.io/v1alpha1","metadata":{"name":"10.49.27.205","creationTimestamp":null,"labels":{"ipType":"internalIP"}},"spec":{"key":"p1","attribute":"{\"NodeName\":\"node2\",\"Uid\":\"xx2\"}","policy":2,"subnet":"10.49.27.0/24","updateTime":null}}`, + `{"kind":"FloatingIP","apiVersion":"galaxy.k8s.io/v1alpha1","metadata":{"name":"10.49.27.216","creationTimestamp":null,"labels":{"ipType":"internalIP"}},"spec":{"key":"p1","attribute":"{\"NodeName\":\"node2\",\"Uid\":\"xx2\"}","policy":2,"subnet":"10.49.27.0/24","updateTime":null}}`); err != nil { t.Fatal(err) } // reserve again, should not succeed @@ -213,7 +215,7 @@ func TestCRDReserveIP(t *testing.T) { } } -func TestCRDRelease(t *testing.T) { +func TestRelease(t *testing.T) { ipam := createTestCrdIPAM(t) testRelease(t, ipam) if err := checkFIP(ipam, pod1CRD); err != nil { @@ -221,7 +223,7 @@ func TestCRDRelease(t *testing.T) { } } -func TestCRDReleaseIPs(t *testing.T) { +func TestReleaseIPs(t *testing.T) { ipam := createTestCrdIPAM(t) testReleaseIPs(t, ipam) if err := checkFIP(ipam, pod2CRD); err != nil { @@ -229,12 +231,12 @@ func TestCRDReleaseIPs(t *testing.T) { } } -func TestCRDByKeyword(t *testing.T) { +func TestByKeyword(t *testing.T) { ipam := createTestCrdIPAM(t) testByKeyword(t, ipam) } -func TestCRDByPrefix(t *testing.T) { +func TestByPrefix(t *testing.T) { ipam := createTestCrdIPAM(t) testByPrefix(t, ipam) } @@ -280,11 +282,18 @@ func testReleaseIPs(t *testing.T, ipam IPAM) { func testByKeyword(t *testing.T, ipam IPAM) { now := time.Now() - allocateSomeIPs(t, ipam) fips, err := ipam.ByKeyword("od") if err != nil { t.Fatal(err) } + if len(fips) != 0 { + t.Fatal(len(fips)) + } + allocateSomeIPs(t, ipam) + fips, err = ipam.ByKeyword("od") + if err != nil { + t.Fatal(err) + } if len(fips) != 2 { t.Fatal(len(fips)) } @@ -501,3 +510,179 @@ func TestAllocateInMultipleSubnet(t *testing.T) { t.Fatalf("expect allocated ip both from %s and %s", node7FIPSubnet, node6FIPSubnet) } } + +func TestAllocateInSubnetsAndIPRange(t *testing.T) { + for i, testCase := range []struct { + nodeSubnet *net.IPNet + ipranges string + expectIPs []string // skip check ips if expectIPs is empty + expectFIPSubnet *net.IPNet + expectError error + }{ + {nodeSubnet: node1IPNet, ipranges: "", expectFIPSubnet: node1FIPSubnet}, + {nodeSubnet: node1IPNet, ipranges: `[["10.49.27.216~10.49.27.218"]]`, expectFIPSubnet: node1FIPSubnet}, + {nodeSubnet: node1IPNet, ipranges: `[["10.49.27.217~10.49.27.218"],["10.49.27.217~10.49.27.218"]]`, + expectFIPSubnet: node1FIPSubnet, expectIPs: []string{"10.49.27.217", "10.49.27.218"}}, + {nodeSubnet: node1IPNet, ipranges: `[["10.49.27.205", "10.49.27.218"]]`, expectFIPSubnet: node1FIPSubnet, + expectIPs: []string{"10.49.27.205"}}, + {nodeSubnet: node1IPNet, ipranges: `[["10.49.27.205"],["10.49.27.218"]]`, expectFIPSubnet: node1FIPSubnet, + expectIPs: []string{"10.49.27.205", "10.49.27.218"}}, + {nodeSubnet: node1IPNet, ipranges: `[["10.49.27.216"],["10.49.27.217"],["10.49.27.218"]]`, + expectFIPSubnet: node1FIPSubnet, + expectIPs: []string{"10.49.27.216", "10.49.27.217", "10.49.27.218"}}, + // node1IPNet has not 10.50.0.1 + {nodeSubnet: node1IPNet, ipranges: `[["10.49.27.216"],["10.50.0.1"]]`, expectError: ErrNoEnoughIP}, + // node2IPNet has not 10.49.27.216 + {nodeSubnet: node2IPNet, ipranges: `[["10.49.27.216"]]`, expectError: ErrNoEnoughIP}, + } { + ipam := createTestCrdIPAM(t) + var ipranges [][]nets.IPRange + if testCase.ipranges != "" { + if err := json.Unmarshal([]byte(testCase.ipranges), &ipranges); err != nil { + t.Fatalf("case %d: %v", i, err) + } + } + ips, err := ipam.AllocateInSubnetsAndIPRange("p1", testCase.nodeSubnet, ipranges, Attr{}) + if err != nil { + if testCase.expectError != nil && testCase.expectError == err && len(ips) == 0 { + continue + } + t.Fatalf("case %d: %v", i, err) + } + for i := range ips { + if !testCase.expectFIPSubnet.Contains(ips[i]) { + t.Fatalf("case %d, expect %s contains allocatedIP %s", i, testCase.expectFIPSubnet, ips[i]) + } + } + if len(testCase.expectIPs) == 0 { + continue + } + if len(testCase.expectIPs) != len(ips) { + t.Fatalf("case %d, expect %v, real %v", i, testCase.expectIPs, ips) + } + for i := range ips { + if ips[i].String() != testCase.expectIPs[i] { + t.Fatalf("case %d, expect %v, real %v", i, testCase.expectIPs, ips) + } + } + } +} + +func TestAllocateInSubnetsAndIPRange2(t *testing.T) { + // check if AllocateInSubnetsAndIPRange allocates all ips or nothing + ipam := createTestCrdIPAM(t) + ipranges := [][]nets.IPRange{{*nets.ParseIPRange("10.49.27.216")}, {*nets.ParseIPRange("10.50.0.1")}} + ips, err := ipam.AllocateInSubnetsAndIPRange("p1", node1IPNet, ipranges, Attr{}) + if err != ErrNoEnoughIP || len(ips) != 0 { + t.Fatalf("%v, %v", ips, err) + } + if fip, err := ipam.ByIP(net.ParseIP("10.49.27.216")); err != nil { + t.Fatal(err) + } else if fip.Key != "" { + t.Fatal() + } + ipranges = [][]nets.IPRange{{*nets.ParseIPRange("10.49.27.216")}, {*nets.ParseIPRange("10.49.27.218")}} + // check if attr is correct + ips, err = ipam.AllocateInSubnetsAndIPRange("p1", node1IPNet, ipranges, + Attr{Policy: constant.ReleasePolicyImmutable, NodeName: "node2", Uid: "xx1"}) + if err != nil || len(ips) != 2 { + t.Fatalf("%v, %v", ips, err) + } + for _, ipStr := range []string{"10.49.27.216", "10.49.27.218"} { + fip, err := ipam.ByIP(net.ParseIP(ipStr)) + if err != nil { + t.Fatal(err) + } + if fip.Policy != uint16(constant.ReleasePolicyImmutable) || fip.NodeName != "node2" || fip.PodUid != "xx1" { + t.Fatal(fip) + } + } +} + +func TestByKeyAndIPRanges(t *testing.T) { + ipam := createTestCrdIPAM(t) + ipranges := [][]nets.IPRange{{*nets.ParseIPRange("10.49.27.216")}, {*nets.ParseIPRange("10.49.27.218")}} + _, err := ipam.AllocateInSubnetsAndIPRange("p1", node1IPNet, ipranges, Attr{}) + if err != nil { + t.Fatal() + } + ipInfos, err := ipam.ByKeyAndIPRanges("p1", ipranges) + if err != nil || len(ipInfos) != 2 || + ipInfos[0].IP.String() != "10.49.27.216" || ipInfos[1].IP.String() != "10.49.27.218" || + ipInfos[0].IPInfo.Gateway.String() != "10.49.27.1" || ipInfos[1].IPInfo.Gateway.String() != "10.49.27.1" { + t.Fatalf("%v, %v", ipInfos, err) + } + // test if ipranges is nil, result should be the same + ipInfos1, err := ipam.ByKeyAndIPRanges("p1", nil) + if err != nil || len(ipInfos1) != 2 { + t.Fatalf("%v, %v", ipInfos1, err) + } + if ipInfos1[0].IP.String() == "10.49.27.218" { + ipInfos1[0], ipInfos1[1] = ipInfos1[1], ipInfos1[0] + } + if ipInfos1[0].IP.String() != "10.49.27.216" || ipInfos1[1].IP.String() != "10.49.27.218" || + ipInfos1[0].IPInfo.Gateway.String() != "10.49.27.1" || ipInfos1[1].IPInfo.Gateway.String() != "10.49.27.1" { + t.Fatalf("%v, %v", ipInfos1, err) + } + // test if there is unallocated ips in ipranges + ipranges = append([][]nets.IPRange{{*nets.ParseIPRange("10.49.27.205~10.49.27.214")}}, ipranges...) + ipInfos2, err := ipam.ByKeyAndIPRanges("p1", ipranges) + if err != nil || len(ipInfos2) != 3 || + ipInfos2[0] != nil || + ipInfos2[1].IP.String() != "10.49.27.216" || ipInfos2[2].IP.String() != "10.49.27.218" || + ipInfos2[1].IPInfo.Gateway.String() != "10.49.27.1" || ipInfos2[2].IPInfo.Gateway.String() != "10.49.27.1" { + t.Fatalf("%v, %v", ipInfos2, err) + } + // test if iprange is small + ipInfos, err = ipam.ByKeyAndIPRanges("p1", [][]nets.IPRange{{*nets.ParseIPRange("10.49.27.218")}}) + if err != nil || len(ipInfos) != 1 || + ipInfos[0].IP.String() != "10.49.27.218" || + ipInfos[0].IPInfo.Gateway.String() != "10.49.27.1" { + t.Fatalf("%v, %v", ipInfos, err) + } +} + +func TestNodeSubnetsByKeyAndIPRanges(t *testing.T) { + allNodeSubnetsSet := sets.NewString() + for i := range allNodeSubnet { + allNodeSubnetsSet.Insert(allNodeSubnet[i].String()) + } + ipam := createTestCrdIPAM(t) + for i, testCase := range []struct { + ipranges string + expectSubnets []*net.IPNet + }{ + {expectSubnets: allNodeSubnet}, + {ipranges: `[["10.49.27.216~10.49.27.218"]]`, expectSubnets: []*net.IPNet{node1IPNet}}, + {ipranges: `[["10.49.27.216"],["10.49.27.218"]]`, expectSubnets: []*net.IPNet{node1IPNet}}, + {ipranges: `[["10.49.27.216", "10.173.13.10~10.173.13.13"]]`, + expectSubnets: []*net.IPNet{node1IPNet, node2IPNet}}, + {ipranges: `[["10.49.27.216", "10.173.13.10~10.173.13.13"],["10.173.13.13"]]`, + expectSubnets: []*net.IPNet{node2IPNet}}, + {ipranges: `[["10.49.27.216", "10.173.13.10~10.173.13.13", "10.180.154.2"]]`, + expectSubnets: []*net.IPNet{node1IPNet, node2IPNet, node3IPNet}}, + {ipranges: `[["10.49.27.216"],["10.173.13.10~10.173.13.13"]]`, expectSubnets: []*net.IPNet{}}, + {ipranges: `[["10.0.70.3~10.0.70.20"]]`, + expectSubnets: []*net.IPNet{node5IPNet1, node5IPNet2}}, + {ipranges: `[["10.0.70.3", "10.0.80.2"]]`, + expectSubnets: []*net.IPNet{node5IPNet1, node5IPNet2, node6IPNet1, node6IPNet2}}, + } { + var ipranges [][]nets.IPRange + if testCase.ipranges != "" { + if err := json.Unmarshal([]byte(testCase.ipranges), &ipranges); err != nil { + t.Fatalf("case %d: %v", i, err) + } + } + subnets, err := ipam.NodeSubnetsByIPRanges(ipranges) + if err != nil { + t.Fatalf("case %d: %v", i, err) + } + expectSubnets := sets.NewString() + for i := range testCase.expectSubnets { + expectSubnets.Insert(testCase.expectSubnets[i].String()) + } + if !reflect.DeepEqual(subnets.List(), expectSubnets.List()) { + t.Fatalf("case %d, expect %v, real %v", i, testCase.expectSubnets, subnets) + } + } +} diff --git a/pkg/ipam/floatingip/store_crd.go b/pkg/ipam/floatingip/store_crd.go index 939040b9..135d3057 100644 --- a/pkg/ipam/floatingip/store_crd.go +++ b/pkg/ipam/floatingip/store_crd.go @@ -84,7 +84,9 @@ func assign(spec *v1alpha1.FloatingIP, f *FloatingIP) error { return err } spec.Spec.Attribute = string(data) - spec.Spec.Subnet = strings.Join(f.Subnets.List(), ",") + if f.pool != nil { + spec.Spec.Subnet = strings.Join(f.pool.nodeSubnets.List(), ",") + } spec.Spec.UpdateTime = metav1.NewTime(f.UpdatedAt) return nil } diff --git a/pkg/ipam/floatingip/store_crd_test.go b/pkg/ipam/floatingip/store_crd_test.go index 6c47c053..2f39861c 100644 --- a/pkg/ipam/floatingip/store_crd_test.go +++ b/pkg/ipam/floatingip/store_crd_test.go @@ -24,7 +24,6 @@ import ( "time" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "tkestack.io/galaxy/pkg/api/galaxy/constant" ) @@ -37,7 +36,6 @@ func TestAddFloatingIPEventByUser(t *testing.T) { fip := &FloatingIP{ IP: net.ParseIP("10.49.27.205"), Key: "pod2", - Subnets: sets.NewString("subnet1"), Policy: 0, UpdatedAt: time.Now(), } @@ -78,7 +76,7 @@ func waitFor(ipam *crdIpam, ip net.IP, key string, expectReserveLabel bool, expe if hasReserveLabel != expectReserveLabel { return false, fmt.Errorf("expect has reserve label %v, got %v", expectReserveLabel, hasReserveLabel) } - subnetStr := strings.Join(searched.Subnets.List(), ",") + subnetStr := strings.Join(searched.pool.nodeSubnets.List(), ",") if subnetStr != expectSubnetStr { return false, fmt.Errorf("expect subnet %v, got %v", expectSubnetStr, subnetStr) } diff --git a/pkg/ipam/schedulerplugin/bind.go b/pkg/ipam/schedulerplugin/bind.go index b16a4ba2..d5584efc 100644 --- a/pkg/ipam/schedulerplugin/bind.go +++ b/pkg/ipam/schedulerplugin/bind.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" glog "k8s.io/klog" "tkestack.io/galaxy/pkg/api/galaxy/constant" @@ -32,6 +33,7 @@ import ( "tkestack.io/galaxy/pkg/ipam/floatingip" "tkestack.io/galaxy/pkg/ipam/metrics" "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" + "tkestack.io/galaxy/pkg/utils/nets" ) // Bind binds a new floatingip or reuse an old one to pod @@ -78,8 +80,7 @@ func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error { } return false, nil } - glog.Infof("bind pod %s to %s with ip %v", keyObj.KeyInDB, args.Node, - bindAnnotation[constant.ExtendedCNIArgsAnnotation]) + glog.Infof("bind pod %s to %s with %s", keyObj.KeyInDB, args.Node, string(data)) return true, nil }); err != nil { if apierrors.IsNotFound(err1) { @@ -95,43 +96,49 @@ func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error { } func (p *FloatingIPPlugin) allocateIP(key string, nodeName string, pod *corev1.Pod) (*constant.CniArgs, error) { - var how string cniArgs, err := getPodCniArgs(pod) if err != nil { return nil, err } - ipInfos, err := p.ipam.ByKeyAndIPRanges(key, cniArgs.RequestIPRange) + ipranges := cniArgs.RequestIPRange + ipInfos, err := p.ipam.ByKeyAndIPRanges(key, ipranges) if err != nil { return nil, fmt.Errorf("failed to query floating ip by key %s: %v", key, err) } - started := time.Now() + if len(ipranges) == 0 && len(ipInfos) > 0 { + // reuse only one if requesting only one ip + ipInfos = ipInfos[:1] + } + var unallocatedIPRange [][]nets.IPRange // those does not have allocated ips + reservedIPs := sets.NewString() + for i := range ipInfos { + if ipInfos[i] == nil { + unallocatedIPRange = append(unallocatedIPRange, ipranges[i]) + } else { + reservedIPs.Insert(ipInfos[i].IP.String()) + } + } policy := parseReleasePolicy(&pod.ObjectMeta) attr := floatingip.Attr{Policy: policy, NodeName: nodeName, Uid: string(pod.UID)} - if len(ipInfos) != 0 { - how = "reused" - for _, ipInfo := range ipInfos { - // check if uid missmatch, if we delete a statfulset/tapp and creates a same name statfulset/tapp immediately, - // galaxy-ipam may receive bind event for new pod early than deleting event for old pod - if ipInfo.FIP.PodUid != "" && ipInfo.FIP.PodUid != string(pod.GetUID()) { - return nil, fmt.Errorf("waiting for delete event of %s before reuse this ip", key) - } + for _, ipInfo := range ipInfos { + // check if uid missmatch, if we delete a statfulset/tapp and creates a same name statfulset/tapp immediately, + // galaxy-ipam may receive bind event for new pod early than deleting event for old pod + if ipInfo != nil && ipInfo.PodUid != "" && ipInfo.PodUid != string(pod.GetUID()) { + return nil, fmt.Errorf("waiting for delete event of %s before reuse this ip", key) } - } else { + } + if len(unallocatedIPRange) > 0 || len(ipInfos) == 0 { subnet, err := p.queryNodeSubnet(nodeName) if err != nil { return nil, err } - if _, err := p.ipam.AllocateInSubnetsAndIPRange(key, subnet, cniArgs.RequestIPRange, attr); err != nil { + if _, err := p.ipam.AllocateInSubnetsAndIPRange(key, subnet, unallocatedIPRange, attr); err != nil { return nil, err } - how = "allocated" - ipInfos, err = p.ipam.ByKeyAndIPRanges(key, cniArgs.RequestIPRange) + ipInfos, err = p.ipam.ByKeyAndIPRanges(key, ipranges) if err != nil { return nil, fmt.Errorf("failed to query floating ip by key %s: %v", key, err) } - if len(ipInfos) == 0 { - return nil, fmt.Errorf("nil floating ip for key %s: %v", key, err) - } } for _, ipInfo := range ipInfos { glog.Infof("AssignIP nodeName %s, ip %s, key %s", nodeName, ipInfo.IPInfo.IP.IP.String(), key) @@ -142,20 +149,22 @@ func (p *FloatingIPPlugin) allocateIP(key string, nodeName string, pod *corev1.P // do not rollback allocated ip return nil, fmt.Errorf("failed to assign ip %s to %s: %v", ipInfo.IPInfo.IP.IP.String(), key, err) } - if how == "reused" { - glog.Infof("pod %s reused %s, updating attr to %v", key, ipInfo.IPInfo.IP.String(), attr) + if reservedIPs.Has(ipInfo.IP.String()) { + glog.Infof("%s reused %s, updating attr to %v", key, ipInfo.IPInfo.IP.String(), attr) if err := p.ipam.UpdateAttr(key, ipInfo.IPInfo.IP.IP, attr); err != nil { return nil, fmt.Errorf("failed to update floating ip release policy: %v", err) } } } - var ips []string + var allocatedIPs []string var ret []constant.IPInfo for _, ipInfo := range ipInfos { - ips = append(ips, ipInfo.IPInfo.IP.String()) + if !reservedIPs.Has(ipInfo.IP.String()) { + allocatedIPs = append(allocatedIPs, ipInfo.IP.String()) + } ret = append(ret, ipInfo.IPInfo) } - glog.Infof("started at %d %s ips %s, attr %v for %s", started.UnixNano(), how, ips, attr, key) + glog.Infof("%s reused ips %v, allocated ips %v, attr %v", key, reservedIPs.List(), allocatedIPs, attr) cniArgs.Common.IPInfos = ret return &cniArgs, nil } @@ -176,9 +185,9 @@ func (p *FloatingIPPlugin) unbind(pod *corev1.Pod) error { } for _, ipInfo := range ipInfos { ipStr := ipInfo.IPInfo.IP.IP.String() - glog.Infof("UnAssignIP nodeName %s, ip %s, key %s", ipInfo.FIP.NodeName, ipStr, key) + glog.Infof("UnAssignIP nodeName %s, ip %s, key %s", ipInfo.NodeName, ipStr, key) if err = p.cloudProviderUnAssignIP(&rpc.UnAssignIPRequest{ - NodeName: ipInfo.FIP.NodeName, + NodeName: ipInfo.NodeName, IPAddress: ipStr, }); err != nil { return fmt.Errorf("failed to unassign ip %s from %s: %v", ipStr, key, err) diff --git a/pkg/ipam/schedulerplugin/bind_test.go b/pkg/ipam/schedulerplugin/bind_test.go index c7d51e14..0f875ec6 100644 --- a/pkg/ipam/schedulerplugin/bind_test.go +++ b/pkg/ipam/schedulerplugin/bind_test.go @@ -17,6 +17,7 @@ package schedulerplugin import ( + "encoding/json" "fmt" "net" "reflect" @@ -25,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" fakeV1 "k8s.io/client-go/kubernetes/typed/core/v1/fake" "tkestack.io/galaxy/pkg/api/galaxy/constant" @@ -33,6 +35,7 @@ import ( "tkestack.io/galaxy/pkg/ipam/floatingip" . "tkestack.io/galaxy/pkg/ipam/schedulerplugin/testing" schedulerplugin_util "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" + "tkestack.io/galaxy/pkg/utils/nets" ) func TestBind(t *testing.T) { @@ -42,30 +45,11 @@ func TestBind(t *testing.T) { if err != nil { t.Fatalf("checkBind error %v", err) } - fakePods := fipPlugin.PluginFactoryArgs.Client.CoreV1().Pods(pod.Namespace).(*fakeV1.FakePods) - - actualBinding, err := fakePods.GetBinding(pod.GetName()) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - return - } - str, err := constant.MarshalCniArgs([]constant.IPInfo{fipInfo.IPInfo}) - if err != nil { + if err := checkBinding(fipPlugin, pod, &constant.CniArgs{Common: constant.CommonCniArgs{ + IPInfos: []constant.IPInfo{fipInfo.IPInfo}, + }}, node3); err != nil { t.Fatal(err) } - expect := &corev1.Binding{ - ObjectMeta: v1.ObjectMeta{ - Namespace: pod.Namespace, Name: pod.Name, - Annotations: map[string]string{ - constant.ExtendedCNIArgsAnnotation: str}}, - Target: corev1.ObjectReference{ - Kind: "Node", - Name: node3, - }, - } - if !reflect.DeepEqual(expect, actualBinding) { - t.Fatalf("Binding did not match expectation, expect %v, actual %v", expect, actualBinding) - } } func TestAllocateIP(t *testing.T) { @@ -90,14 +74,14 @@ func TestAllocateIP(t *testing.T) { if err != nil { t.Fatal(err) } - if fip.FIP.Policy != uint16(constant.ReleasePolicyImmutable) { - t.Fatal(fip.FIP.Policy) + if fip.Policy != uint16(constant.ReleasePolicyImmutable) { + t.Fatal(fip.Policy) } - if fip.FIP.NodeName != node4 { - t.Fatal(fip.FIP.NodeName) + if fip.NodeName != node4 { + t.Fatal(fip.NodeName) } - if fip.FIP.PodUid != string(pod.UID) { - t.Fatal(fip.FIP.PodUid) + if fip.PodUid != string(pod.UID) { + t.Fatal(fip.PodUid) } } @@ -153,12 +137,16 @@ func TestUnBind(t *testing.T) { if err := drainNode(fipPlugin, node3Subnet, expectIP); err != nil { t.Fatal(err) } - fakeCP := &FakeCloudProvider{ExpectIP: expectIP.String(), ExpectNode: node3} + fakeCP := NewFakeCloudProvider() fipPlugin.cloudProvider = fakeCP fipInfo, err := checkBind(fipPlugin, pod1, node3, keyObj.KeyInDB, node3Subnet) if err != nil { t.Fatal(err) } + if err := checkFakeCloudProviderState(fakeCP, map[string]string{"10.49.27.205": node3}, + map[string]string{}); err != nil { + t.Fatal(err) + } str, err := constant.MarshalCniArgs([]constant.IPInfo{fipInfo.IPInfo}) if err != nil { t.Fatal(err) @@ -168,8 +156,9 @@ func TestUnBind(t *testing.T) { if err := fipPlugin.unbind(pod1); err != nil { t.Fatal(err) } - if !fakeCP.InvokedAssignIP || !fakeCP.InvokedUnAssignIP { - t.Fatal() + if err := checkFakeCloudProviderState(fakeCP, map[string]string{"10.49.27.205": node3}, + map[string]string{"10.49.27.205": node3}); err != nil { + t.Fatal(err) } } @@ -236,10 +225,176 @@ func TestReleaseIPOfFinishedPod(t *testing.T) { t.Fatalf("case %d: %v", i, err) } if err := wait.Poll(time.Microsecond*10, time.Second*30, func() (done bool, err error) { - return checkIPKey(fipPlugin.ipam, fipInfo.FIP.IP.String(), "") == nil, nil + return checkIPKey(fipPlugin.ipam, fipInfo.IP.String(), "") == nil, nil }); err != nil { t.Fatalf("case %d: %v", i, err) } }() } } + +func TestBindUnBindRequestIPRange(t *testing.T) { + // create a pod request for two ips + request := `{"request_ip_range":[["10.49.27.205"],["10.49.27.217"]]}` + cniArgs, err := constant.UnmarshalCniArgs(request) + if err != nil { + t.Fatal(err) + } + pod := CreateStatefulSetPod("pod1-0", "ns1", cniArgsAnnotation(request)) + podKey, _ := schedulerplugin_util.FormatKey(pod) + fipPlugin, stopChan, _ := createPluginTestNodes(t, pod) + cp := NewFakeCloudProvider() + fipPlugin.cloudProvider = cp + defer func() { stopChan <- struct{}{} }() + if err := checkBindForIPRanges(fipPlugin, pod, node3, cniArgs, "10.49.27.205", "10.49.27.217"); err != nil { + t.Fatal(err) + } + if err := fipPlugin.unbind(pod); err != nil { + t.Fatal(err) + } + if _, err := checkByKeyAndIPRanges(fipPlugin, podKey.KeyInDB, cniArgs.RequestIPRange); err != nil { + t.Fatal(err) + } + expectAssigned := map[string]string{"10.49.27.205": node3, "10.49.27.217": node3} + // both assigned ips should be unassigned + if err := checkFakeCloudProviderState(cp, expectAssigned, expectAssigned); err != nil { + t.Fatal(err) + } +} + +func TestFilterBindRequestIPRange(t *testing.T) { + // test for both allocated iprange and unallocated iprange + request := `{"request_ip_range":[["10.49.27.205"]]}` + cniArgs, err := constant.UnmarshalCniArgs(request) + if err != nil { + t.Fatal(err) + } + pod := CreateStatefulSetPod("pod1-0", "ns1", cniArgsAnnotation(request)) + fipPlugin, stopChan, _ := createPluginTestNodes(t, pod) + cp := NewFakeCloudProvider() + fipPlugin.cloudProvider = cp + defer func() { stopChan <- struct{}{} }() + if err := checkBindForIPRanges(fipPlugin, pod, node3, cniArgs, "10.49.27.205"); err != nil { + t.Fatal(err) + } + request = `{"request_ip_range":[["10.49.27.205"],["10.49.27.218"]]}` + cniArgs, err = constant.UnmarshalCniArgs(request) + if err != nil { + t.Fatal(err) + } + pod.Annotations = cniArgsAnnotation(request) + if _, err := fipPlugin.Client.CoreV1().Pods(pod.Namespace).Update(pod); err != nil { + t.Fatal(err) + } + // wait for lister updates + if err := wait.Poll(time.Millisecond*100, time.Minute, func() (done bool, err error) { + pod1, err := fipPlugin.PodLister.Pods(pod.Namespace).Get(pod.Name) + if err != nil { + return false, nil + } + return pod1.Annotations[constant.ExtendedCNIArgsAnnotation] == + pod.Annotations[constant.ExtendedCNIArgsAnnotation], nil + }); err != nil { + t.Fatal() + } + if err := checkBindForIPRanges(fipPlugin, pod, node3, cniArgs, "10.49.27.205", "10.49.27.218"); err != nil { + t.Fatal(err) + } +} + +func checkBindForIPRanges(fipPlugin *FloatingIPPlugin, pod *corev1.Pod, node string, cniArgs *constant.CniArgs, + expectIPs ...string) error { + podKey, _ := schedulerplugin_util.FormatKey(pod) + if err := fipPlugin.Bind(&schedulerapi.ExtenderBindingArgs{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + Node: node, + }); err != nil { + return err + } + fipInfos, err := checkByKeyAndIPRanges(fipPlugin, podKey.KeyInDB, cniArgs.RequestIPRange, expectIPs...) + if err != nil { + return err + } + var ipInfos []constant.IPInfo + for i := range fipInfos { + ipInfos = append(ipInfos, fipInfos[i].IPInfo) + } + cniArgs.Common.IPInfos = ipInfos + if err := checkBinding(fipPlugin, pod, cniArgs, node); err != nil { + return err + } + if fipPlugin.cloudProvider != nil { + expectAssigned := map[string]string{} + for _, ip := range expectIPs { + expectAssigned[ip] = node + } + if err := checkFakeCloudProviderState(fipPlugin.cloudProvider.(*FakeCloudProvider), expectAssigned, + map[string]string{}); err != nil { + return err + } + } + return nil +} + +func checkFakeCloudProviderState(cp *FakeCloudProvider, expectAssigned, expectUnassigned map[string]string) error { + if !reflect.DeepEqual(cp.Assigned, expectAssigned) { + return fmt.Errorf("fake cloud provider assigned missmatch, expect %v, real %v", expectAssigned, + cp.Assigned) + } + if !reflect.DeepEqual(cp.UnAssigned, expectUnassigned) { + return fmt.Errorf("fake cloud provider unassigned missmatch, expect %v, real %v", expectAssigned, + cp.Assigned) + } + return nil +} + +func checkBinding(fipPlugin *FloatingIPPlugin, pod *corev1.Pod, expectCniArgs *constant.CniArgs, + expectNode string) error { + actualBinding, err := fipPlugin.PluginFactoryArgs.Client.CoreV1().Pods(pod.Namespace).(*fakeV1.FakePods). + GetBinding(pod.GetName()) + if err != nil { + return err + } + data, err := json.Marshal(expectCniArgs) + if err != nil { + return err + } + expect := &corev1.Binding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: pod.Namespace, Name: pod.Name, + Annotations: cniArgsAnnotation(string(data))}, + Target: corev1.ObjectReference{ + Kind: "Node", + Name: expectNode, + }, + } + if !reflect.DeepEqual(expect, actualBinding) { + return fmt.Errorf("binding did not match expectation, expect %v, actual %v", expect, actualBinding) + } + return nil +} + +func checkByKeyAndIPRanges(fipPlugin *FloatingIPPlugin, key string, ipranges [][]nets.IPRange, + expectIP ...string) ([]*floatingip.FloatingIPInfo, error) { + fipInfos, err := fipPlugin.ipam.ByKeyAndIPRanges(key, ipranges) + if err != nil { + return nil, err + } + realIPs := sets.NewString() + for i := range fipInfos { + if fipInfos[i] != nil { + realIPs.Insert(fipInfos[i].IPInfo.IP.IP.String()) + } + } + realIPList := realIPs.List() + if len(realIPList) != len(expectIP) { + return nil, fmt.Errorf("expect %v, real %v", expectIP, realIPList) + } + for i := range expectIP { + if expectIP[i] != realIPList[i] { + return nil, fmt.Errorf("expect %v, real %v", expectIP, realIPList) + } + } + return fipInfos, nil +} diff --git a/pkg/ipam/schedulerplugin/deployment_test.go b/pkg/ipam/schedulerplugin/deployment_test.go index 9cf290a8..c36bfae3 100644 --- a/pkg/ipam/schedulerplugin/deployment_test.go +++ b/pkg/ipam/schedulerplugin/deployment_test.go @@ -50,7 +50,7 @@ func TestDpReleasePolicy(t *testing.T) { if err := fipPlugin.unbind(pod); err != nil { t.Fatalf("case %d, err %v", i, err) } - if err := checkIPKey(fipPlugin.ipam, fip.FIP.IP.String(), testCase.expectKeyFunc(keyObj)); err != nil { + if err := checkIPKey(fipPlugin.ipam, fip.IP.String(), testCase.expectKeyFunc(keyObj)); err != nil { t.Fatalf("case %d, err %v", i, err) } }() diff --git a/pkg/ipam/schedulerplugin/filter.go b/pkg/ipam/schedulerplugin/filter.go index 8cec311f..2ef966d9 100644 --- a/pkg/ipam/schedulerplugin/filter.go +++ b/pkg/ipam/schedulerplugin/filter.go @@ -29,6 +29,7 @@ import ( "tkestack.io/galaxy/pkg/ipam/floatingip" "tkestack.io/galaxy/pkg/ipam/metrics" "tkestack.io/galaxy/pkg/ipam/schedulerplugin/util" + "tkestack.io/galaxy/pkg/utils/nets" ) // Filter marks nodes which have no available ips as FailedNodes @@ -64,7 +65,8 @@ func (p *FloatingIPPlugin) Filter(pod *corev1.Pod, nodes []corev1.Node) ([]corev for i := range filteredNodes { nodeNames[i] = filteredNodes[i].Name } - glog.V(5).Infof("filtered nodes %v failed nodes %v", nodeNames, failedNodesMap) + glog.V(5).Infof("filtered nodes %v failed nodes %v for %s_%s", nodeNames, failedNodesMap, + pod.Namespace, pod.Name) } metrics.ScheduleLatency.WithLabelValues("filter").Observe(time.Since(start).Seconds()) return filteredNodes, failedNodesMap, nil @@ -80,14 +82,42 @@ func (p *FloatingIPPlugin) getSubnet(pod *corev1.Pod) (sets.String, error) { if err != nil { return nil, err } + ipranges := cniArgs.RequestIPRange // first check if exists an already allocated ip for this pod - subnets, err := p.ipam.NodeSubnetsByKeyAndIPRanges(keyObj.KeyInDB, cniArgs.RequestIPRange) + ipInfos, err := p.ipam.ByKeyAndIPRanges(keyObj.KeyInDB, ipranges) if err != nil { return nil, fmt.Errorf("failed to query by key %s: %v", keyObj.KeyInDB, err) } - if len(subnets) > 0 { - glog.V(3).Infof("%s already have an allocated ip in subnets %v", keyObj.KeyInDB, subnets) - return subnets, nil + allocatedSubnets := sets.NewString() + if len(ipranges) == 0 { + if len(ipInfos) > 0 { + glog.V(3).Infof("%s already have an allocated ip %s in subnets %v", keyObj.KeyInDB, + ipInfos[0].IP.String(), ipInfos[0].NodeSubnets) + return ipInfos[0].NodeSubnets, nil + } + } else { + var unallocatedIPRange [][]nets.IPRange // those does not have allocated ips + var ips []string + for i := range ipInfos { + if ipInfos[i] == nil { + unallocatedIPRange = append(unallocatedIPRange, ipranges[i]) + } else { + ips = append(ips, ipInfos[i].IP.String()) + if allocatedSubnets.Len() == 0 { + allocatedSubnets.Insert(ipInfos[i].NodeSubnets.UnsortedList()...) + } else { + allocatedSubnets = allocatedSubnets.Intersection(ipInfos[i].NodeSubnets) + } + } + } + if len(unallocatedIPRange) == 0 { + glog.V(3).Infof("%s already have allocated ips %v with intersection subnets %v", + keyObj.KeyInDB, ips, allocatedSubnets) + return allocatedSubnets, nil + } + glog.V(3).Infof("%s have allocated ips %v with intersection subnets %v, but also unallocated "+ + "ip ranges %v", keyObj.KeyInDB, ips, allocatedSubnets, unallocatedIPRange) + ipranges = unallocatedIPRange } policy := parseReleasePolicy(&pod.ObjectMeta) if !keyObj.Deployment() && !keyObj.StatefulSet() && !keyObj.TApp() && policy != constant.ReleasePolicyPodDelete { @@ -103,10 +133,13 @@ func (p *FloatingIPPlugin) getSubnet(pod *corev1.Pod) (sets.String, error) { // Lock to make checking available subnets and allocating reserved ip atomic defer p.LockDpPool(keyObj.PoolPrefix())() } - subnetSet, reserve, err := p.getAvailableSubnet(keyObj, policy, replicas, isPoolSizeDefined, cniArgs.RequestIPRange) + subnetSet, reserve, err := p.getAvailableSubnet(keyObj, policy, replicas, isPoolSizeDefined, ipranges) if err != nil { return nil, err } + if allocatedSubnets.Len() > 0 { + subnetSet = subnetSet.Intersection(allocatedSubnets) + } if (reserve || isPoolSizeDefined) && subnetSet.Len() > 0 { // Since bind is in a different goroutine than filter in scheduler, we can't ensure this pod got binded // before the next one got filtered to ensure max size of allocated ips. diff --git a/pkg/ipam/schedulerplugin/filter_test.go b/pkg/ipam/schedulerplugin/filter_test.go index f929274c..aff253df 100644 --- a/pkg/ipam/schedulerplugin/filter_test.go +++ b/pkg/ipam/schedulerplugin/filter_test.go @@ -151,14 +151,14 @@ type filterCase struct { testPod *corev1.Pod expectErr error expectFiltererd, expectFailed []string - preHook func() error + preHook func(*filterCase) error postHook func() error } // #lizard forgives func checkFilterCase(fipPlugin *FloatingIPPlugin, testCase filterCase, nodes []corev1.Node) error { if testCase.preHook != nil { - if err := testCase.preHook(); err != nil { + if err := testCase.preHook(&testCase); err != nil { return fmt.Errorf("preHook failed: %v", err) } } @@ -200,7 +200,7 @@ func TestFilterForDeploymentIPPool(t *testing.T) { { // test bind gets the right key, i.e. dp_ns1_dp_dp-xxx-yyy, and filter gets reserved node testPod: pod, expectFiltererd: []string{node4}, expectFailed: []string{drainedNode, nodeHasNoIP, node3}, - preHook: func() error { + preHook: func(*filterCase) error { return fipPlugin.ipam.AllocateSpecificIP(podKey.KeyInDB, net.ParseIP("10.173.13.2"), floatingip.Attr{Policy: constant.ReleasePolicyNever}) }, @@ -208,7 +208,7 @@ func TestFilterForDeploymentIPPool(t *testing.T) { { // test unbind gets the right key, i.e. pool__pool1_, and filter on pod2 gets reserved node and key is updating to pod2, i.e. dp_ns1_dp2_dp2-abc-def testPod: pod2, expectFiltererd: []string{node4}, expectFailed: []string{drainedNode, nodeHasNoIP, node3}, - preHook: func() error { + preHook: func(*filterCase) error { // because replicas = 1, ip will be reserved if err := fipPlugin.unbind(pod); err != nil { t.Fatal(err) @@ -277,3 +277,83 @@ func checkFailed(realFailed schedulerapi.FailedNodesMap, failed ...string) error } return nil } + +func TestFilterRequestIPRange(t *testing.T) { + // node3 can allocate 10.49.27.0/24 + // node5 can allocate 10.49.27.0/24, 10.0.80.0/24, 10.0.81.0/24 + // node6 can allocate 10.0.80.0/24 + n5, n6 := CreateNode("node5", nil, "10.49.28.3"), CreateNode("node6", nil, "10.49.29.3") + node5, node6 := n5.Name, n6.Name + fipPlugin, stopChan, nodes := createPluginTestNodes(t, &n5, &n6) + defer func() { stopChan <- struct{}{} }() + for i, testCase := range []filterCase{ + { + testPod: CreateStatefulSetPod("pod1-0", "ns1", + cniArgsAnnotation(`{"request_ip_range":[["10.49.27.205"],["10.49.27.216~10.49.27.218"]]}`)), + expectFiltererd: []string{node3}, + expectFailed: []string{drainedNode, nodeHasNoIP, node4, node5, node6}, + }, + // create a pod request for two ips, only nodes in 10.49.28.0/24() can meet the requests + { + testPod: CreateStatefulSetPod("pod1-0", "ns1", + cniArgsAnnotation(`{"request_ip_range":[["10.0.80.2~10.0.80.4"],["10.0.81.2"]]}`)), + expectFiltererd: []string{node5}, + expectFailed: []string{drainedNode, nodeHasNoIP, node3, node4, node6}, + }, + // create a pod request for 10.0.80.2~10.0.80.4, both node5 and node6 meet the requests + { + testPod: CreateStatefulSetPod("pod1-0", "ns1", + cniArgsAnnotation(`{"request_ip_range":[["10.0.80.2~10.0.80.4"]]}`)), + expectFiltererd: []string{node5, node6}, + expectFailed: []string{drainedNode, nodeHasNoIP, node3, node4}, + }, + // create a pod request for two ips, 10.49.27.205 and one of 10.0.80.2~10.0.80.4, no node meet the requests + { + testPod: CreateStatefulSetPod("pod1-0", "ns1", + cniArgsAnnotation(`{"request_ip_range":[["10.49.27.205"],["10.0.80.2~10.0.80.4"]]}`)), + expectFiltererd: []string{}, + expectFailed: []string{drainedNode, nodeHasNoIP, node3, node4, node5, node6}, + }, + // create a pod request for 10.49.27.205 or 10.0.80.2~10.0.80.4, node3, node5, node6 meet the requests + { + testPod: CreateStatefulSetPod("pod1-0", "ns1", + cniArgsAnnotation(`{"request_ip_range":[["10.49.27.205","10.0.80.2~10.0.80.4"]]}`)), + expectFiltererd: []string{node3, node5, node6}, + expectFailed: []string{drainedNode, nodeHasNoIP, node4}, + }, + // TODO allocates multiple ips in the overlapped ip range + //{ + // testPod: CreateStatefulSetPod("pod1-0", "ns1", + // cniArgsAnnotation(`{"request_ip_range":[["10.49.27.205","10.0.80.2~10.0.80.4"],["10.49.27.205","10.0.80.2~10.0.80.4"]]}`)), + // expectFiltererd: []string{node5, node6}, + // expectFailed: []string{drainedNode, nodeHasNoIP, node3, node4}, + //}, + { + testPod: CreateStatefulSetPod("pod1-0", "ns1", + cniArgsAnnotation(`{"request_ip_range":[["10.49.27.216~10.49.27.218","10.0.80.2~10.0.80.4"],["10.49.27.216~10.49.27.218","10.0.80.2~10.0.80.4"]]}`)), + expectFiltererd: []string{node3, node5, node6}, + expectFailed: []string{drainedNode, nodeHasNoIP, node4}, + }, + { + testPod: CreateStatefulSetPod("pod1-0", "ns1", + cniArgsAnnotation(`{"request_ip_range":[["10.49.27.216~10.49.27.218","10.0.80.2~10.0.80.4"],["10.49.27.216~10.49.27.218","10.0.80.2~10.0.80.4"]]}`)), + preHook: func(fc *filterCase) error { + podKey, _ := schedulerplugin_util.FormatKey(fc.testPod) + if _, err := fipPlugin.allocateIP(podKey.KeyInDB, node6, fc.testPod); err != nil { + t.Fatal(err) + } + return nil + }, + expectFiltererd: []string{node5, node6}, + expectFailed: []string{drainedNode, nodeHasNoIP, node3, node4}, + }, + } { + if err := checkFilterCase(fipPlugin, testCase, nodes); err != nil { + t.Fatalf("case %d: %v", i, err) + } + } +} + +func cniArgsAnnotation(poolName string) map[string]string { + return map[string]string{constant.ExtendedCNIArgsAnnotation: poolName} +} diff --git a/pkg/ipam/schedulerplugin/floatingip_plugin_test.go b/pkg/ipam/schedulerplugin/floatingip_plugin_test.go index dabf527f..0c59fbd5 100644 --- a/pkg/ipam/schedulerplugin/floatingip_plugin_test.go +++ b/pkg/ipam/schedulerplugin/floatingip_plugin_test.go @@ -77,6 +77,11 @@ func createPluginTestNodes(t *testing.T, objs ...runtime.Object) (*FloatingIPPlu if err := drainNode(fipPlugin, subnet, nil); err != nil { t.Fatal(err) } + for i := range objs { + if node, ok := objs[i].(*corev1.Node); ok { + nodes = append(nodes, *node) + } + } return fipPlugin, stopChan, nodes } diff --git a/pkg/ipam/schedulerplugin/ipam.go b/pkg/ipam/schedulerplugin/ipam.go index 7dfbd718..6cc23fa2 100644 --- a/pkg/ipam/schedulerplugin/ipam.go +++ b/pkg/ipam/schedulerplugin/ipam.go @@ -79,7 +79,7 @@ func (p *FloatingIPPlugin) getAvailableSubnet(keyObj *util.KeyObj, policy consta return nil, false, fmt.Errorf("request ip ranges for deployment pod with release " + "policy other than ReleasePolicyPodDelete is not supported") } - var ips []floatingip.FloatingIP + var ips []*floatingip.FloatingIPInfo poolPrefix := keyObj.PoolPrefix() poolAppPrefix := keyObj.PoolAppPrefix() ips, err = p.ipam.ByPrefix(poolPrefix) @@ -100,7 +100,7 @@ func (p *FloatingIPPlugin) getAvailableSubnet(keyObj *util.KeyObj, policy consta } } } else { - unusedSubnetSet.Insert(ip.Subnets.UnsortedList()...) + unusedSubnetSet.Insert(ip.NodeSubnets.UnsortedList()...) } } glog.V(4).Infof("keyObj %v, unusedSubnetSet %v, usedCount %d, replicas %d, isPoolSizeDefined %v", keyObj, @@ -117,7 +117,7 @@ func (p *FloatingIPPlugin) getAvailableSubnet(keyObj *util.KeyObj, policy consta return unusedSubnetSet, true, nil } } - if subnets, err = p.ipam.NodeSubnetsByKeyAndIPRanges("", ipranges); err != nil { + if subnets, err = p.ipam.NodeSubnetsByIPRanges(ipranges); err != nil { err = fmt.Errorf("failed to query allocatable subnet: %v", err) return } @@ -132,7 +132,7 @@ func (p *FloatingIPPlugin) releaseIP(key string, reason string) error { } m := map[string]string{} for i := range ipInfos { - m[ipInfos[i].FIP.IP.String()] = ipInfos[i].FIP.Key + m[ipInfos[i].IP.String()] = ipInfos[i].Key } released, unreleased, err := p.ipam.ReleaseIPs(m) if err != nil { diff --git a/pkg/ipam/schedulerplugin/resync.go b/pkg/ipam/schedulerplugin/resync.go index 102422a7..9cb6095c 100644 --- a/pkg/ipam/schedulerplugin/resync.go +++ b/pkg/ipam/schedulerplugin/resync.go @@ -47,9 +47,7 @@ type resyncObj struct { func (p *FloatingIPPlugin) resyncPod() error { glog.V(4).Infof("resync pods+") defer glog.V(4).Infof("resync pods-") - resyncMeta := &resyncMeta{ - allocatedIPs: make(map[string]resyncObj), - } + resyncMeta := &resyncMeta{} if err := p.fetchChecklist(resyncMeta); err != nil { return err } @@ -58,7 +56,7 @@ func (p *FloatingIPPlugin) resyncPod() error { } type resyncMeta struct { - allocatedIPs map[string]resyncObj // allocated ips from galaxy pool + allocatedIPs []resyncObj // allocated ips from galaxy pool } func (p *FloatingIPPlugin) fetchChecklist(meta *resyncMeta) error { @@ -79,14 +77,15 @@ func (p *FloatingIPPlugin) fetchChecklist(meta *resyncMeta) error { glog.Warningf("unexpected key: %s", fip.Key) continue } - meta.allocatedIPs[fip.Key] = resyncObj{keyObj: keyObj, fip: fip} + meta.allocatedIPs = append(meta.allocatedIPs, resyncObj{keyObj: keyObj, fip: fip.FloatingIP}) } return nil } // #lizard forgives func (p *FloatingIPPlugin) resyncAllocatedIPs(meta *resyncMeta) { - for key, obj := range meta.allocatedIPs { + for _, obj := range meta.allocatedIPs { + key := obj.keyObj.KeyInDB func() { defer p.lockPod(obj.keyObj.PodName, obj.keyObj.Namespace)() if p.podRunning(obj.keyObj.PodName, obj.keyObj.Namespace, obj.fip.PodUid) { diff --git a/pkg/ipam/schedulerplugin/statefulset_test.go b/pkg/ipam/schedulerplugin/statefulset_test.go index d11aa606..f5f49c37 100644 --- a/pkg/ipam/schedulerplugin/statefulset_test.go +++ b/pkg/ipam/schedulerplugin/statefulset_test.go @@ -60,7 +60,7 @@ func TestStsReleasePolicy(t *testing.T) { if err := fipPlugin.unbind(pod); err != nil { t.Fatalf("case %d, err %v", i, err) } - if err := checkIPKey(fipPlugin.ipam, fip.FIP.IP.String(), testCase.expectKeyFunc(keyObj)); err != nil { + if err := checkIPKey(fipPlugin.ipam, fip.IP.String(), testCase.expectKeyFunc(keyObj)); err != nil { t.Fatalf("case %d, err %v", i, err) } }() diff --git a/pkg/utils/nets/ip.go b/pkg/utils/nets/ip.go index 55a16070..f6572e4d 100644 --- a/pkg/utils/nets/ip.go +++ b/pkg/utils/nets/ip.go @@ -149,6 +149,22 @@ func ParseIPRange(ipr string) *IPRange { } } +func (ipr IPRange) MarshalJSON() ([]byte, error) { + return json.Marshal(ipr.String()) +} + +func (ipr *IPRange) UnmarshalJSON(data []byte) error { + if len(data) < 3 { + return fmt.Errorf("bad IPRange format %s", string(data)) + } + r := ParseIPRange(string(data[1 : len(data)-1])) + if r == nil { + return fmt.Errorf("bad IPRange format %s", string(data)) + } + *ipr = *r + return nil +} + // SparseSubnet represents a sparse subnet type SparseSubnet struct { IPRanges []IPRange `json:"ranges"`