Skip to content

Commit

Permalink
feat: pod can use multiple nic with the same subnet
Browse files Browse the repository at this point in the history
  • Loading branch information
@fanriming
fanriming committed Oct 28, 2021
1 parent 85a9559 commit 850e421
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 206 deletions.
7 changes: 4 additions & 3 deletions pkg/controller/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ func (c *Controller) InitIPAM() error {
for _, podNet := range podNets {
_, _, _, err := c.ipam.GetStaticAddress(
fmt.Sprintf("%s/%s", pod.Namespace, pod.Name),
ovs.PodNameToPortName(pod.Name, pod.Namespace, podNet.ProviderName),
pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)],
pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)],
pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)], false)
Expand All @@ -326,15 +327,15 @@ func (c *Controller) InitIPAM() error {
} else {
ipamKey = fmt.Sprintf("node-%s", ip.Spec.PodName)
}
if _, _, _, err = c.ipam.GetStaticAddress(ipamKey, ip.Spec.IPAddress, ip.Spec.MacAddress, ip.Spec.Subnet, false); err != nil {
if _, _, _, err = c.ipam.GetStaticAddress(ipamKey, ip.Name, ip.Spec.IPAddress, ip.Spec.MacAddress, ip.Spec.Subnet, false); err != nil {
klog.Errorf("failed to init IPAM from IP CR %s: %v", ip.Name, err)
}
for i := range ip.Spec.AttachSubnets {
if i == len(ip.Spec.AttachIPs) || i == len(ip.Spec.AttachMacs) {
klog.Errorf("attachment IP/MAC of IP CR %s is invalid", ip.Name)
break
}
if _, _, _, err = c.ipam.GetStaticAddress(ipamKey, ip.Spec.AttachIPs[i], ip.Spec.AttachMacs[i], ip.Spec.AttachSubnets[i], false); err != nil {
if _, _, _, err = c.ipam.GetStaticAddress(ipamKey, ip.Name, ip.Spec.AttachIPs[i], ip.Spec.AttachMacs[i], ip.Spec.AttachSubnets[i], false); err != nil {
klog.Errorf("failed to init IPAM from IP CR %s: %v", ip.Name, err)
}
}
Expand All @@ -348,7 +349,7 @@ func (c *Controller) InitIPAM() error {
for _, node := range nodes {
if node.Annotations[util.AllocatedAnnotation] == "true" {
portName := fmt.Sprintf("node-%s", node.Name)
v4IP, v6IP, _, err := c.ipam.GetStaticAddress(portName, node.Annotations[util.IpAddressAnnotation],
v4IP, v6IP, _, err := c.ipam.GetStaticAddress(portName, portName, node.Annotations[util.IpAddressAnnotation],
node.Annotations[util.MacAddressAnnotation],
node.Annotations[util.LogicalSwitchAnnotation], true)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,15 +250,15 @@ func (c *Controller) handleAddNode(key string) error {
var v4IP, v6IP, mac string
portName := fmt.Sprintf("node-%s", key)
if node.Annotations[util.AllocatedAnnotation] == "true" && node.Annotations[util.IpAddressAnnotation] != "" && node.Annotations[util.MacAddressAnnotation] != "" {
v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(portName, node.Annotations[util.IpAddressAnnotation],
v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(portName, portName, node.Annotations[util.IpAddressAnnotation],
node.Annotations[util.MacAddressAnnotation],
node.Annotations[util.LogicalSwitchAnnotation], true)
if err != nil {
klog.Errorf("failed to alloc static ip addrs for node %v: %v", node.Name, err)
return err
}
} else {
v4IP, v6IP, mac, err = c.ipam.GetRandomAddress(portName, c.config.NodeSwitch, nil)
v4IP, v6IP, mac, err = c.ipam.GetRandomAddress(portName, portName, c.config.NodeSwitch, nil)
if err != nil {
klog.Errorf("failed to alloc random ip addrs for node %v: %v", node.Name, err)
return err
Expand Down
24 changes: 8 additions & 16 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,16 +402,6 @@ func (c *Controller) getPodKubeovnNets(pod *v1.Pod) ([]*kubeovnNet, error) {
})
}

for i, net1 := range podNets {
if i >= len(podNets)-1 {
break
}
for _, net2 := range podNets[i+1:] {
if net1.Subnet.Name == net2.Subnet.Name {
return nil, fmt.Errorf("subnet conflict, the same subnet should not be attached repeatedly")
}
}
}
return podNets, nil
}

Expand Down Expand Up @@ -1073,7 +1063,8 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)] == "" {
var skippedAddrs []string
for {
ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, podNet.Subnet.Name, skippedAddrs)
nicName := ovs.PodNameToPortName(pod.Name, pod.Namespace, podNet.ProviderName)
ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, nicName, podNet.Subnet.Name, skippedAddrs)
if err != nil {
return "", "", "", err
}
Expand All @@ -1095,10 +1086,11 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
}
}

nicName := ovs.PodNameToPortName(pod.Name, pod.Namespace, podNet.ProviderName)
// Static allocate
if pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)] != "" {
ipStr := pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)]
return c.acquireStaticAddress(key, ipStr, macStr, podNet.Subnet.Name, podNet.AllowLiveMigration)
return c.acquireStaticAddress(key, nicName, ipStr, macStr, podNet.Subnet.Name, podNet.AllowLiveMigration)
}

// IPPool allocate
Expand All @@ -1113,7 +1105,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
klog.Errorf("static address %s for %s has been assigned", staticIP, key)
continue
}
if v4IP, v6IP, mac, err := c.acquireStaticAddress(key, staticIP, macStr, podNet.Subnet.Name, podNet.AllowLiveMigration); err == nil {
if v4IP, v6IP, mac, err := c.acquireStaticAddress(key, nicName, staticIP, macStr, podNet.Subnet.Name, podNet.AllowLiveMigration); err == nil {
return v4IP, v6IP, mac, nil
} else {
klog.Errorf("acquire address %s for %s failed, %v", staticIP, key, err)
Expand All @@ -1124,7 +1116,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
numStr := strings.Split(pod.Name, "-")[numIndex]
index, _ := strconv.Atoi(numStr)
if index < len(ipPool) {
return c.acquireStaticAddress(key, ipPool[index], macStr, podNet.Subnet.Name, podNet.AllowLiveMigration)
return c.acquireStaticAddress(key, nicName, ipPool[index], macStr, podNet.Subnet.Name, podNet.AllowLiveMigration)
}
}
klog.Errorf("alloc address for %s failed, return NoAvailableAddress", key)
Expand All @@ -1143,7 +1135,7 @@ func generatePatchPayload(annotations map[string]string, op string) []byte {
return []byte(fmt.Sprintf(patchPayloadTemplate, op, raw))
}

func (c *Controller) acquireStaticAddress(key, ip, mac, subnet string, liveMigration bool) (string, string, string, error) {
func (c *Controller) acquireStaticAddress(key, nicName, ip, mac, subnet string, liveMigration bool) (string, string, string, error) {
var v4IP, v6IP string
var err error
for _, ipStr := range strings.Split(ip, ",") {
Expand All @@ -1152,7 +1144,7 @@ func (c *Controller) acquireStaticAddress(key, ip, mac, subnet string, liveMigra
}
}

if v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(key, ip, mac, subnet, !liveMigration); err != nil {
if v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(key, nicName, ip, mac, subnet, !liveMigration); err != nil {
klog.Errorf("failed to get static ip %v, mac %v, subnet %v, err %v", ip, mac, subnet, err)
return "", "", "", err
}
Expand Down
50 changes: 27 additions & 23 deletions pkg/ipam/ipam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func NewIPAM() *IPAM {
}
}

func (ipam *IPAM) GetRandomAddress(podName, subnetName string, skippedAddrs []string) (string, string, string, error) {
func (ipam *IPAM) GetRandomAddress(podName, nicName, subnetName string, skippedAddrs []string) (string, string, string, error) {
ipam.mutex.RLock()
defer ipam.mutex.RUnlock()

Expand All @@ -45,12 +45,12 @@ func (ipam *IPAM) GetRandomAddress(podName, subnetName string, skippedAddrs []st
return "", "", "", ErrNoAvailable
}

v4IP, v6IP, mac, err := subnet.GetRandomAddress(podName, skippedAddrs)
v4IP, v6IP, mac, err := subnet.GetRandomAddress(podName, nicName, skippedAddrs)
klog.Infof("allocate v4 %s v6 %s mac %s for %s", v4IP, v6IP, mac, podName)
return string(v4IP), string(v6IP), mac, err
}

func (ipam *IPAM) GetStaticAddress(podName, ip, mac, subnetName string, checkConflict bool) (string, string, string, error) {
func (ipam *IPAM) GetStaticAddress(podName, nicName, ip, mac, subnetName string, checkConflict bool) (string, string, string, error) {
ipam.mutex.RLock()
defer ipam.mutex.RUnlock()
if subnet, ok := ipam.Subnets[subnetName]; !ok {
Expand All @@ -60,13 +60,13 @@ func (ipam *IPAM) GetStaticAddress(podName, ip, mac, subnetName string, checkCon
var err error
var ipAddr IP
for _, ipStr := range strings.Split(ip, ",") {
ipAddr, mac, err = subnet.GetStaticAddress(podName, IP(ipStr), mac, false, checkConflict)
ipAddr, mac, err = subnet.GetStaticAddress(podName, nicName, IP(ipStr), mac, false, checkConflict)
if err != nil {
return "", "", "", err
}
ips = append(ips, ipAddr)
}
ips, err = checkAndAppendIpsForDual(ips, podName, subnet)
ips, err = checkAndAppendIpsForDual(ips, podName, nicName, subnet)
if err != nil {
klog.Errorf("failed to append allocate ip %v mac %s for %s", ips, mac, podName)
return "", "", "", err
Expand All @@ -87,7 +87,7 @@ func (ipam *IPAM) GetStaticAddress(podName, ip, mac, subnetName string, checkCon
return "", "", "", ErrNoAvailable
}

func checkAndAppendIpsForDual(ips []IP, podName string, subnet *Subnet) ([]IP, error) {
func checkAndAppendIpsForDual(ips []IP, podName string, nicName string, subnet *Subnet) ([]IP, error) {
// IP Address for dual-stack should be format of 'IPv4,IPv6'
if subnet.Protocol != kubeovnv1.ProtocolDual || len(ips) == 2 {
return ips, nil
Expand All @@ -98,10 +98,10 @@ func checkAndAppendIpsForDual(ips []IP, podName string, subnet *Subnet) ([]IP, e
var err error
if util.CheckProtocol(string(ips[0])) == kubeovnv1.ProtocolIPv4 {
newIps = ips
_, ipAddr, _, err = subnet.getV6RandomAddress(podName, nil)
_, ipAddr, _, err = subnet.getV6RandomAddress(podName, nicName, nil)
newIps = append(newIps, ipAddr)
} else if util.CheckProtocol(string(ips[0])) == kubeovnv1.ProtocolIPv6 {
ipAddr, _, _, err = subnet.getV4RandomAddress(podName, nil)
ipAddr, _, _, err = subnet.getV4RandomAddress(podName, nicName, nil)
newIps = append(newIps, ipAddr)
newIps = append(newIps, ips...)
}
Expand Down Expand Up @@ -156,9 +156,10 @@ func (ipam *IPAM) AddOrUpdateSubnet(name, cidrStr string, excludeIps []string) e
lastIP, _ := util.LastIP(v4cidrStr)
subnet.V4FreeIPList = IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}}
subnet.joinFreeWithReserve()
for podName, ip := range subnet.V4PodToIP {
mac := subnet.PodToMac[podName]
if _, _, err := subnet.GetStaticAddress(podName, ip, mac, true, true); err != nil {
for nicName, ip := range subnet.V4NicToIP {
mac := subnet.NicToMac[nicName]
podName := subnet.V4IPToPod[ip]
if _, _, err := subnet.GetStaticAddress(podName, nicName, ip, mac, true, true); err != nil {
klog.Errorf("%s address not in subnet %s new cidr %s: %v", podName, name, cidrStr, err)
}
}
Expand All @@ -171,9 +172,10 @@ func (ipam *IPAM) AddOrUpdateSubnet(name, cidrStr string, excludeIps []string) e
lastIP, _ := util.LastIP(v6cidrStr)
subnet.V6FreeIPList = IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}}
subnet.joinFreeWithReserve()
for podName, ip := range subnet.V6PodToIP {
mac := subnet.PodToMac[podName]
if _, _, err := subnet.GetStaticAddress(podName, ip, mac, true, true); err != nil {
for nicName, ip := range subnet.V6NicToIP {
mac := subnet.NicToMac[nicName]
podName := subnet.V6IPToPod[ip]
if _, _, err := subnet.GetStaticAddress(podName, nicName, ip, mac, true, true); err != nil {
klog.Errorf("%s address not in subnet %s new cidr %s: %v", podName, name, cidrStr, err)
}
}
Expand Down Expand Up @@ -202,15 +204,17 @@ func (ipam *IPAM) GetPodAddress(podName string) []*SubnetAddress {
defer ipam.mutex.RUnlock()
addresses := []*SubnetAddress{}
for _, subnet := range ipam.Subnets {
v4IP, v6IP, mac, protocol := subnet.GetPodAddress(podName)
switch protocol {
case kubeovnv1.ProtocolIPv4:
addresses = append(addresses, &SubnetAddress{Subnet: subnet, Ip: string(v4IP), Mac: mac})
case kubeovnv1.ProtocolIPv6:
addresses = append(addresses, &SubnetAddress{Subnet: subnet, Ip: string(v6IP), Mac: mac})
case kubeovnv1.ProtocolDual:
addresses = append(addresses, &SubnetAddress{Subnet: subnet, Ip: string(v4IP), Mac: mac})
addresses = append(addresses, &SubnetAddress{Subnet: subnet, Ip: string(v6IP), Mac: mac})
for _, nicName := range subnet.PodToNicList[podName] {
v4IP, v6IP, mac, protocol := subnet.GetPodAddress(podName, nicName)
switch protocol {
case kubeovnv1.ProtocolIPv4:
addresses = append(addresses, &SubnetAddress{Subnet: subnet, Ip: string(v4IP), Mac: mac})
case kubeovnv1.ProtocolIPv6:
addresses = append(addresses, &SubnetAddress{Subnet: subnet, Ip: string(v6IP), Mac: mac})
case kubeovnv1.ProtocolDual:
addresses = append(addresses, &SubnetAddress{Subnet: subnet, Ip: string(v4IP), Mac: mac})
addresses = append(addresses, &SubnetAddress{Subnet: subnet, Ip: string(v6IP), Mac: mac})
}
}
}
return addresses
Expand Down

0 comments on commit 850e421

Please sign in to comment.