Skip to content

Commit

Permalink
add V4/V6UsingIPRange and V4/V6AvailableIPRange in subnet status (#2268)
Browse files Browse the repository at this point in the history 
* add V4/V6UsingIPRange and V4/V6AvailableIPRange in subnet status

* fix when restart deployment may cause subnetstatus not update

* add case for restart deployment
  • Loading branch information
@changluyi
changluyi committed Jan 30, 2023
1 parent 0c74034 commit 805f83e
Show file tree
Hide file tree
Showing 9 changed files with 486 additions and 7 deletions.
8 changes: 8 additions & 0 deletions dist/images/install.sh
View file
Expand Up @@ -1354,6 +1354,14 @@ spec:
type: string
u2oInterconnectionIP:
type: string
v4usingIPrange:
type: string
v4availableIPrange:
type: string
v6usingIPrange:
type: string
v6availableIPrange:
type: string
conditions:
type: array
items:
Expand Down
8 changes: 8 additions & 0 deletions kubeovn-helm/templates/kube-ovn-crd.yaml
View file
Expand Up @@ -1138,6 +1138,14 @@ spec:
type: string
u2oInterconnectionIP:
type: string
v4usingIPrange:
type: string
v4availableIPrange:
type: string
v6usingIPrange:
type: string
v6availableIPrange:
type: string
conditions:
type: array
items:
Expand Down
4 changes: 4 additions & 0 deletions pkg/apis/kubeovn/v1/types.go
View file
Expand Up @@ -181,9 +181,13 @@ type SubnetStatus struct {
Conditions []SubnetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`

V4AvailableIPs float64 `json:"v4availableIPs"`
V4AvailableIPRange string `json:"v4availableIPrange"`
V4UsingIPs float64 `json:"v4usingIPs"`
V4UsingIPRange string `json:"v4usingIPrange"`
V6AvailableIPs float64 `json:"v6availableIPs"`
V6AvailableIPRange string `json:"v6availableIPrange"`
V6UsingIPs float64 `json:"v6usingIPs"`
V6UsingIPRange string `json:"v6usingIPrange"`
ActivateGateway string `json:"activateGateway"`
DHCPv4OptionsUUID string `json:"dhcpV4OptionsUUID"`
DHCPv6OptionsUUID string `json:"dhcpV6OptionsUUID"`
Expand Down
34 changes: 31 additions & 3 deletions pkg/controller/subnet.go
View file
Expand Up @@ -1568,17 +1568,28 @@ func calcDualSubnetStatusIP(subnet *kubeovnv1.Subnet, c *Controller) error {
v6availableIPs = 0
}

v4UsingIPStr, v6UsingIPStr, v4AvailableIPStr, v6AvailableIPStr := c.ipam.GetSubnetIPRangeString(subnet.Name)

if subnet.Status.V4AvailableIPs == v4availableIPs &&
subnet.Status.V6AvailableIPs == v6availableIPs &&
subnet.Status.V4UsingIPs == usingIPs &&
subnet.Status.V6UsingIPs == usingIPs {
subnet.Status.V6UsingIPs == usingIPs &&
subnet.Status.V4UsingIPRange == v4UsingIPStr &&
subnet.Status.V6UsingIPRange == v6UsingIPStr &&
subnet.Status.V4AvailableIPRange == v4AvailableIPStr &&
subnet.Status.V6AvailableIPRange == v6AvailableIPStr {
return nil
}

subnet.Status.V4AvailableIPs = v4availableIPs
subnet.Status.V6AvailableIPs = v6availableIPs
subnet.Status.V4UsingIPs = usingIPs
subnet.Status.V6UsingIPs = usingIPs
subnet.Status.V4UsingIPRange = v4UsingIPStr
subnet.Status.V6UsingIPRange = v6UsingIPStr
subnet.Status.V4AvailableIPRange = v4AvailableIPStr
subnet.Status.V6AvailableIPRange = v6AvailableIPStr

bytes, err := subnet.Status.Bytes()
if err != nil {
return err
Expand Down Expand Up @@ -1626,28 +1637,45 @@ func calcSubnetStatusIP(subnet *kubeovnv1.Subnet, c *Controller) error {
availableIPs = 0
}

cachedFields := [4]float64{
v4UsingIPStr, v6UsingIPStr, v4AvailableIPStr, v6AvailableIPStr := c.ipam.GetSubnetIPRangeString(subnet.Name)
cachedFloatFields := [4]float64{
subnet.Status.V4AvailableIPs,
subnet.Status.V4UsingIPs,
subnet.Status.V6AvailableIPs,
subnet.Status.V6UsingIPs,
}
cachedStringFields := [4]string{
subnet.Status.V4UsingIPRange,
subnet.Status.V4AvailableIPRange,
subnet.Status.V6UsingIPRange,
subnet.Status.V6AvailableIPRange,
}

if subnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 {
subnet.Status.V4AvailableIPs = availableIPs
subnet.Status.V4UsingIPs = usingIPs
subnet.Status.V4UsingIPRange = v4UsingIPStr
subnet.Status.V4AvailableIPRange = v4AvailableIPStr
subnet.Status.V6AvailableIPs = 0
subnet.Status.V6UsingIPs = 0
} else {
subnet.Status.V6AvailableIPs = availableIPs
subnet.Status.V6UsingIPs = usingIPs
subnet.Status.V6UsingIPRange = v6UsingIPStr
subnet.Status.V6AvailableIPRange = v6AvailableIPStr
subnet.Status.V4AvailableIPs = 0
subnet.Status.V4UsingIPs = 0
}
if cachedFields == [4]float64{
if cachedFloatFields == [4]float64{
subnet.Status.V4AvailableIPs,
subnet.Status.V4UsingIPs,
subnet.Status.V6AvailableIPs,
subnet.Status.V6UsingIPs,
} && cachedStringFields == [4]string{
subnet.Status.V4UsingIPRange,
subnet.Status.V4AvailableIPRange,
subnet.Status.V6UsingIPRange,
subnet.Status.V6AvailableIPRange,
} {
return nil
}
Expand Down
13 changes: 13 additions & 0 deletions pkg/ipam/ip.go
View file
@@ -1,6 +1,7 @@
package ipam

import (
"fmt"
"math/big"
"strings"

Expand Down Expand Up @@ -50,6 +51,18 @@ func (iprl IPRangeList) Contains(ip IP) bool {
return false
}

func (iprl IPRangeList) IpRangetoString() string {
var ipRangeString []string
for _, ipr := range iprl {
if ipr.Start.Equal(ipr.End) {
ipRangeString = append(ipRangeString, string(ipr.Start))
} else {
ipRangeString = append(ipRangeString, fmt.Sprintf("%s-%s", ipr.Start, ipr.End))
}
}
return strings.Join(ipRangeString, ",")
}

func splitIPRangeList(iprl IPRangeList, ip IP) (bool, IPRangeList) {
newIPRangeList := []*IPRange{}
split := false
Expand Down
17 changes: 17 additions & 0 deletions pkg/ipam/ipam.go
View file
Expand Up @@ -261,3 +261,20 @@ func (ipam *IPAM) GetSubnetV4Mask(subnetName string) (string, error) {
return "", ErrNoAvailable
}
}

func (ipam *IPAM) GetSubnetIPRangeString(subnetName string) (string, string, string, string) {
ipam.mutex.RLock()
defer ipam.mutex.RUnlock()

var v4UsingIPStr, v6UsingIPStr, v4AvailableIPStr, v6AvailableIPStr string

if subnet, ok := ipam.Subnets[subnetName]; ok {

v4UsingIPStr = subnet.V4UsingIPList.IpRangetoString()
v6UsingIPStr = subnet.V6UsingIPList.IpRangetoString()
v4AvailableIPStr = subnet.V4AvailIPList.IpRangetoString()
v6AvailableIPStr = subnet.V6AvailIPList.IpRangetoString()
}

return v4UsingIPStr, v6UsingIPStr, v4AvailableIPStr, v6AvailableIPStr
}
86 changes: 85 additions & 1 deletion pkg/ipam/subnet.go
View file
Expand Up @@ -20,12 +20,16 @@ type Subnet struct {
V4FreeIPList IPRangeList
V4ReleasedIPList IPRangeList
V4ReservedIPList IPRangeList
V4AvailIPList IPRangeList
V4UsingIPList IPRangeList
V4NicToIP map[string]IP
V4IPToPod map[IP]string
V6CIDR *net.IPNet
V6FreeIPList IPRangeList
V6ReleasedIPList IPRangeList
V6ReservedIPList IPRangeList
V6AvailIPList IPRangeList
V6UsingIPList IPRangeList
V6NicToIP map[string]IP
V6IPToPod map[IP]string
NicToMac map[string]string
Expand Down Expand Up @@ -64,6 +68,8 @@ func NewSubnet(name, cidrStr string, excludeIps []string) (*Subnet, error) {
V4FreeIPList: IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}},
V4ReleasedIPList: IPRangeList{},
V4ReservedIPList: convertExcludeIps(v4ExcludeIps),
V4AvailIPList: IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}},
V4UsingIPList: IPRangeList{},
V4NicToIP: map[string]IP{},
V4IPToPod: map[IP]string{},
V6NicToIP: map[string]IP{},
Expand All @@ -85,6 +91,8 @@ func NewSubnet(name, cidrStr string, excludeIps []string) (*Subnet, error) {
V6FreeIPList: IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}},
V6ReleasedIPList: IPRangeList{},
V6ReservedIPList: convertExcludeIps(v6ExcludeIps),
V6AvailIPList: IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}},
V6UsingIPList: IPRangeList{},
V4NicToIP: map[string]IP{},
V4IPToPod: map[IP]string{},
V6NicToIP: map[string]IP{},
Expand All @@ -109,12 +117,16 @@ func NewSubnet(name, cidrStr string, excludeIps []string) (*Subnet, error) {
V4FreeIPList: IPRangeList{&IPRange{Start: IP(v4FirstIP), End: IP(v4LastIP)}},
V4ReleasedIPList: IPRangeList{},
V4ReservedIPList: convertExcludeIps(v4ExcludeIps),
V4AvailIPList: IPRangeList{&IPRange{Start: IP(v4FirstIP), End: IP(v4LastIP)}},
V4UsingIPList: IPRangeList{},
V4NicToIP: map[string]IP{},
V4IPToPod: map[IP]string{},
V6CIDR: cidrs[1],
V6FreeIPList: IPRangeList{&IPRange{Start: IP(v6FirstIP), End: IP(v6LastIP)}},
V6ReleasedIPList: IPRangeList{},
V6ReservedIPList: convertExcludeIps(v6ExcludeIps),
V6AvailIPList: IPRangeList{&IPRange{Start: IP(v6FirstIP), End: IP(v6LastIP)}},
V6UsingIPList: IPRangeList{},
V6NicToIP: map[string]IP{},
V6IPToPod: map[IP]string{},
MacToPod: map[string]string{},
Expand Down Expand Up @@ -241,6 +253,13 @@ func (subnet *Subnet) getV4RandomAddress(podName, nicName string, mac string, sk
if !part2.Start.GreaterThan(part2.End) {
subnet.V4FreeIPList = append(subnet.V4FreeIPList, part2)
}
if split, NewV4AvailIPRangeList := splitIPRangeList(subnet.V4AvailIPList, ip); split {
subnet.V4AvailIPList = NewV4AvailIPRangeList
}

if merged, NewV4UsingIPRangeList := mergeIPRangeList(subnet.V4UsingIPList, ip); merged {
subnet.V4UsingIPList = NewV4UsingIPRangeList
}

subnet.V4NicToIP[nicName] = ip
subnet.V4IPToPod[ip] = podName
Expand Down Expand Up @@ -302,6 +321,13 @@ func (subnet *Subnet) getV6RandomAddress(podName, nicName string, mac string, sk
if !part2.Start.GreaterThan(part2.End) {
subnet.V6FreeIPList = append(subnet.V6FreeIPList, part2)
}
if split, NewV6AvailIPRangeList := splitIPRangeList(subnet.V6AvailIPList, ip); split {
subnet.V6AvailIPList = NewV6AvailIPRangeList
}

if merged, NewV6UsingIPRangeList := mergeIPRangeList(subnet.V6UsingIPList, ip); merged {
subnet.V6UsingIPList = NewV6UsingIPRangeList
}

subnet.V6NicToIP[nicName] = ip
subnet.V6IPToPod[ip] = podName
Expand All @@ -317,13 +343,35 @@ func (subnet *Subnet) getV6RandomAddress(podName, nicName string, mac string, sk
}

func (subnet *Subnet) GetStaticAddress(podName, nicName string, ip IP, mac string, force bool, checkConflict bool) (IP, string, error) {
var v4, v6 bool
isAllocated := false
subnet.mutex.Lock()
defer func() {
subnet.pushPodNic(podName, nicName)
if isAllocated {
if v4 {
if split, NewV4AvailIPRangeList := splitIPRangeList(subnet.V4AvailIPList, ip); split {
subnet.V4AvailIPList = NewV4AvailIPRangeList
}

if merged, NewV4UsingIPRangeList := mergeIPRangeList(subnet.V4UsingIPList, ip); merged {
subnet.V4UsingIPList = NewV4UsingIPRangeList
}
}

if v6 {
if split, NewV6AvailIPRangeList := splitIPRangeList(subnet.V6AvailIPList, ip); split {
subnet.V6AvailIPList = NewV6AvailIPRangeList
}

if merged, NewV6UsingIPRangeList := mergeIPRangeList(subnet.V6UsingIPList, ip); merged {
subnet.V6UsingIPList = NewV6UsingIPRangeList
}
}
}
subnet.mutex.Unlock()
}()

var v4, v6 bool
if net.ParseIP(string(ip)).To4() != nil {
v4 = subnet.V4CIDR != nil
} else {
Expand Down Expand Up @@ -379,12 +427,14 @@ func (subnet *Subnet) GetStaticAddress(podName, nicName string, ip IP, mac strin
subnet.V4FreeIPList = newFreeList
subnet.V4NicToIP[nicName] = ip
subnet.V4IPToPod[ip] = podName
isAllocated = true
return ip, mac, nil
} else {
if split, newReleasedList := splitIPRangeList(subnet.V4ReleasedIPList, ip); split {
subnet.V4ReleasedIPList = newReleasedList
subnet.V4NicToIP[nicName] = ip
subnet.V4IPToPod[ip] = podName
isAllocated = true
return ip, mac, nil
}
}
Expand Down Expand Up @@ -414,12 +464,14 @@ func (subnet *Subnet) GetStaticAddress(podName, nicName string, ip IP, mac strin
subnet.V6FreeIPList = newFreeList
subnet.V6NicToIP[nicName] = ip
subnet.V6IPToPod[ip] = podName
isAllocated = true
return ip, mac, nil
} else {
if split, newReleasedList := splitIPRangeList(subnet.V6ReleasedIPList, ip); split {
subnet.V6ReleasedIPList = newReleasedList
subnet.V6NicToIP[nicName] = ip
subnet.V6IPToPod[ip] = podName
isAllocated = true
return ip, mac, nil
}
}
Expand Down Expand Up @@ -459,6 +511,14 @@ func (subnet *Subnet) releaseAddr(podName, nicName string) {
subnet.V4ReleasedIPList = newReleasedList
klog.Infof("release v4 %s mac %s for %s, add ip to released list", ip, mac, podName)
}

if merged, NewV4AvailIPRangeList := mergeIPRangeList(subnet.V4AvailIPList, ip); merged {
subnet.V4AvailIPList = NewV4AvailIPRangeList
}

if split, NewV4UsingIPList := splitIPRangeList(subnet.V4UsingIPList, ip); split {
subnet.V4UsingIPList = NewV4UsingIPList
}
}
}
if ip, ok = subnet.V6NicToIP[nicName]; ok {
Expand Down Expand Up @@ -489,6 +549,14 @@ func (subnet *Subnet) releaseAddr(podName, nicName string) {
subnet.V6ReleasedIPList = newReleasedList
klog.Infof("release v6 %s mac %s for %s, add ip to released list", ip, mac, podName)
}

if merged, NewV6AvailIPRangeList := mergeIPRangeList(subnet.V6AvailIPList, ip); merged {
subnet.V6AvailIPList = NewV6AvailIPRangeList
}

if split, NewV6UsingIPList := splitIPRangeList(subnet.V6UsingIPList, ip); split {
subnet.V6UsingIPList = NewV6UsingIPList
}
}
}
}
Expand Down Expand Up @@ -525,6 +593,14 @@ func (subnet *Subnet) joinFreeWithReserve() {
}
}
subnet.V4FreeIPList = newFreeList

newAvailableList := IPRangeList{}
for _, availIpr := range subnet.V4AvailIPList {
if iprl := splitRange(availIpr, reserveIpr); iprl != nil {
newAvailableList = append(newAvailableList, iprl...)
}
}
subnet.V4AvailIPList = newAvailableList
}
}
if protocol == kubeovnv1.ProtocolDual || protocol == kubeovnv1.ProtocolIPv6 {
Expand All @@ -536,6 +612,14 @@ func (subnet *Subnet) joinFreeWithReserve() {
}
}
subnet.V6FreeIPList = newFreeList

newAvailableList := IPRangeList{}
for _, availIpr := range subnet.V6AvailIPList {
if iprl := splitRange(availIpr, reserveIpr); iprl != nil {
newAvailableList = append(newAvailableList, iprl...)
}
}
subnet.V6AvailIPList = newAvailableList
}
}
}
Expand Down

0 comments on commit 805f83e

Please sign in to comment.