Skip to content

Commit

Permalink
refactor and instrument range_allocator cidr_sets
Browse files Browse the repository at this point in the history 
refactor and add the following metrics to the cidr_sets used by the range
allocator:, under the subsystem: node_ipam_controller

cidrset_cidrs_allocations_total
cidrset_cidrs_releases_total
cidrset_usage_cidrs
cidrset_allocation_tries_per_request
  • Loading branch information
@aojea
aojea committed Jun 16, 2020
1 parent f455a14 commit 069707f
Show file tree
Hide file tree
Showing 4 changed files with 369 additions and 53 deletions.
14 changes: 12 additions & 2 deletions pkg/controller/nodeipam/ipam/cidrset/BUILD
View file
Expand Up @@ -10,13 +10,23 @@ go_test(
name = "go_default_test",
srcs = ["cidr_set_test.go"],
embed = [":go_default_library"],
deps = ["//vendor/k8s.io/klog/v2:go_default_library"],
deps = [
"//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
],
)

go_library(
name = "go_default_library",
srcs = ["cidr_set.go"],
srcs = [
"cidr_set.go",
"metrics.go",
],
importpath = "k8s.io/kubernetes/pkg/controller/nodeipam/ipam/cidrset",
deps = [
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
],
)

filegroup(
Expand Down
124 changes: 73 additions & 51 deletions pkg/controller/nodeipam/ipam/cidrset/cidr_set.go
View file
Expand Up @@ -30,13 +30,26 @@ import (
// be allocated from.
type CidrSet struct {
sync.Mutex
clusterCIDR *net.IPNet
clusterIP net.IP
// clusterCIDR is the CIDR assigned to the cluster
clusterCIDR *net.IPNet
// clusterMaskSize is the mask size, in bits, assigned to the cluster
// caches the mask size to avoid the penalty of calling clusterCIDR.Mask.Size()
clusterMaskSize int
maxCIDRs int
nextCandidate int
used big.Int
subNetMaskSize int
// nodeMask is the network mask assigned to the nodes
nodeMask net.IPMask
// nodeMaskSize is the mask size, in bits,assigned to the nodes
// caches the mask size to avoid the penalty of calling nodeMask.Size()
nodeMaskSize int
// maxCIDRs is the maximum number of CIDRs that can be allocated
maxCIDRs int
// allocatedCIDRs counts the number of CIDRs allocated
allocatedCIDRs int
// nextCandidate points to the next CIDR that should be free
nextCandidate int
// used is a bitmap used to track the CIDRs allocated
used big.Int
// label is used to identify the metrics
label string
}

const (
Expand Down Expand Up @@ -64,72 +77,71 @@ var (
// NewCIDRSet creates a new CidrSet.
func NewCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) (*CidrSet, error) {
clusterMask := clusterCIDR.Mask
clusterMaskSize, _ := clusterMask.Size()
clusterMaskSize, bits := clusterMask.Size()

var maxCIDRs int
if (clusterCIDR.IP.To4() == nil) && (subNetMaskSize-clusterMaskSize > clusterSubnetMaxDiff) {
return nil, ErrCIDRSetSubNetTooBig
}

// register CidrSet metrics
registerCidrsetMetrics()

maxCIDRs = 1 << uint32(subNetMaskSize-clusterMaskSize)
return &CidrSet{
clusterCIDR: clusterCIDR,
clusterIP: clusterCIDR.IP,
nodeMask: net.CIDRMask(subNetMaskSize, bits),
clusterMaskSize: clusterMaskSize,
maxCIDRs: maxCIDRs,
subNetMaskSize: subNetMaskSize,
nodeMaskSize: subNetMaskSize,
label: clusterCIDR.String(),
}, nil
}

func (s *CidrSet) indexToCIDRBlock(index int) *net.IPNet {
var ip []byte
var mask int
switch /*v4 or v6*/ {
case s.clusterIP.To4() != nil:
case s.clusterCIDR.IP.To4() != nil:
{
j := uint32(index) << uint32(32-s.subNetMaskSize)
ipInt := (binary.BigEndian.Uint32(s.clusterIP)) | j
ip = make([]byte, 4)
j := uint32(index) << uint32(32-s.nodeMaskSize)
ipInt := (binary.BigEndian.Uint32(s.clusterCIDR.IP)) | j
ip = make([]byte, net.IPv4len)
binary.BigEndian.PutUint32(ip, ipInt)
mask = 32

}
case s.clusterIP.To16() != nil:
case s.clusterCIDR.IP.To16() != nil:
{
// leftClusterIP | rightClusterIP
// 2001:0DB8:1234:0000:0000:0000:0000:0000
const v6NBits = 128
const halfV6NBits = v6NBits / 2
leftClusterIP := binary.BigEndian.Uint64(s.clusterIP[:halfIPv6Len])
rightClusterIP := binary.BigEndian.Uint64(s.clusterIP[halfIPv6Len:])
leftClusterIP := binary.BigEndian.Uint64(s.clusterCIDR.IP[:halfIPv6Len])
rightClusterIP := binary.BigEndian.Uint64(s.clusterCIDR.IP[halfIPv6Len:])

leftIP, rightIP := make([]byte, halfIPv6Len), make([]byte, halfIPv6Len)
ip = make([]byte, net.IPv6len)

if s.subNetMaskSize <= halfV6NBits {
if s.nodeMaskSize <= halfV6NBits {
// We only care about left side IP
leftClusterIP |= uint64(index) << uint(halfV6NBits-s.subNetMaskSize)
leftClusterIP |= uint64(index) << uint(halfV6NBits-s.nodeMaskSize)
} else {
if s.clusterMaskSize < halfV6NBits {
// see how many bits are needed to reach the left side
btl := uint(s.subNetMaskSize - halfV6NBits)
btl := uint(s.nodeMaskSize - halfV6NBits)
indexMaxBit := uint(64 - bits.LeadingZeros64(uint64(index)))
if indexMaxBit > btl {
leftClusterIP |= uint64(index) >> btl
}
}
// the right side will be calculated the same way either the
// subNetMaskSize affects both left and right sides
rightClusterIP |= uint64(index) << uint(v6NBits-s.subNetMaskSize)
rightClusterIP |= uint64(index) << uint(v6NBits-s.nodeMaskSize)
}
binary.BigEndian.PutUint64(leftIP, leftClusterIP)
binary.BigEndian.PutUint64(rightIP, rightClusterIP)

ip = append(leftIP, rightIP...)
mask = 128
binary.BigEndian.PutUint64(ip[:halfIPv6Len], leftClusterIP)
binary.BigEndian.PutUint64(ip[halfIPv6Len:], rightClusterIP)
}
}
return &net.IPNet{
IP: ip,
Mask: net.CIDRMask(s.subNetMaskSize, mask),
Mask: s.nodeMask,
}
}

Expand All @@ -139,22 +151,27 @@ func (s *CidrSet) AllocateNext() (*net.IPNet, error) {
s.Lock()
defer s.Unlock()

nextUnused := -1
for i := 0; i < s.maxCIDRs; i++ {
candidate := (i + s.nextCandidate) % s.maxCIDRs
if s.allocatedCIDRs == s.maxCIDRs {
return nil, ErrCIDRRangeNoCIDRsRemaining
}
candidate := s.nextCandidate
var i int
for i = 0; i < s.maxCIDRs; i++ {
if s.used.Bit(candidate) == 0 {
nextUnused = candidate
break
}
candidate = (candidate + 1) % s.maxCIDRs
}
if nextUnused == -1 {
return nil, ErrCIDRRangeNoCIDRsRemaining
}
s.nextCandidate = (nextUnused + 1) % s.maxCIDRs

s.used.SetBit(&s.used, nextUnused, 1)
s.nextCandidate = (candidate + 1) % s.maxCIDRs
s.used.SetBit(&s.used, candidate, 1)
s.allocatedCIDRs++
// Update metrics
cidrSetAllocations.WithLabelValues(s.label).Inc()
cidrSetAllocationTriesPerRequest.WithLabelValues(s.label).Observe(float64(i))
cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs))

return s.indexToCIDRBlock(nextUnused), nil
return s.indexToCIDRBlock(candidate), nil
}

func (s *CidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err error) {
Expand All @@ -176,10 +193,9 @@ func (s *CidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err
if cidr.IP.To4() == nil {
ipSize = net.IPv6len
}
subNetMask := net.CIDRMask(s.subNetMaskSize, ipSize*8)
begin, err = s.getIndexForCIDR(&net.IPNet{
IP: cidr.IP.Mask(subNetMask),
Mask: subNetMask,
IP: cidr.IP.Mask(s.nodeMask),
Mask: s.nodeMask,
})
if err != nil {
return -1, -1, err
Expand All @@ -197,8 +213,8 @@ func (s *CidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err
binary.BigEndian.PutUint64(ip[net.IPv6len/2:], ipIntRight)
}
end, err = s.getIndexForCIDR(&net.IPNet{
IP: net.IP(ip).Mask(subNetMask),
Mask: subNetMask,
IP: net.IP(ip).Mask(s.nodeMask),
Mask: s.nodeMask,
})
if err != nil {
return -1, -1, err
Expand All @@ -217,7 +233,11 @@ func (s *CidrSet) Release(cidr *net.IPNet) error {
defer s.Unlock()
for i := begin; i <= end; i++ {
s.used.SetBit(&s.used, i, 0)
s.allocatedCIDRs--
cidrSetReleases.WithLabelValues(s.label).Inc()
}

cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs))
return nil
}

Expand All @@ -228,13 +248,15 @@ func (s *CidrSet) Occupy(cidr *net.IPNet) (err error) {
if err != nil {
return err
}

s.Lock()
defer s.Unlock()
for i := begin; i <= end; i++ {
s.used.SetBit(&s.used, i, 1)
s.allocatedCIDRs++
cidrSetAllocations.WithLabelValues(s.label).Inc()
}

cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs))
return nil
}

Expand All @@ -244,19 +266,19 @@ func (s *CidrSet) getIndexForCIDR(cidr *net.IPNet) (int, error) {

func (s *CidrSet) getIndexForIP(ip net.IP) (int, error) {
if ip.To4() != nil {
cidrIndex := (binary.BigEndian.Uint32(s.clusterIP) ^ binary.BigEndian.Uint32(ip.To4())) >> uint32(32-s.subNetMaskSize)
cidrIndex := (binary.BigEndian.Uint32(s.clusterCIDR.IP) ^ binary.BigEndian.Uint32(ip.To4())) >> uint32(32-s.nodeMaskSize)
if cidrIndex >= uint32(s.maxCIDRs) {
return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.subNetMaskSize)
return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.nodeMaskSize)
}
return int(cidrIndex), nil
}
if ip.To16() != nil {
bigIP := big.NewInt(0).SetBytes(s.clusterIP)
bigIP := big.NewInt(0).SetBytes(s.clusterCIDR.IP)
bigIP = bigIP.Xor(bigIP, big.NewInt(0).SetBytes(ip))
cidrIndexBig := bigIP.Rsh(bigIP, uint(net.IPv6len*8-s.subNetMaskSize))
cidrIndexBig := bigIP.Rsh(bigIP, uint(net.IPv6len*8-s.nodeMaskSize))
cidrIndex := cidrIndexBig.Uint64()
if cidrIndex >= uint64(s.maxCIDRs) {
return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.subNetMaskSize)
return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.nodeMaskSize)
}
return int(cidrIndex), nil
}
Expand Down

0 comments on commit 069707f

Please sign in to comment.