Skip to content

Commit

Permalink
do not allocate MAC address when kube-ovn is called as an IPAM plugin (…
Browse files Browse the repository at this point in the history 
…#2816)
  • Loading branch information
@zhangzujian
zhangzujian committed May 17, 2023
1 parent a30daea commit 4f015f6
Show file tree
Hide file tree
Showing 10 changed files with 198 additions and 167 deletions.
16 changes: 8 additions & 8 deletions pkg/controller/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ func (c *Controller) InitIPAM() error {
u2oInterconnName := fmt.Sprintf(util.U2OInterconnName, subnet.Spec.Vpc, subnet.Name)
u2oInterconnLrpName := fmt.Sprintf("%s-%s", subnet.Spec.Vpc, subnet.Name)
if subnet.Status.U2OInterconnectionIP != "" {
if _, _, _, err = c.ipam.GetStaticAddress(u2oInterconnName, u2oInterconnLrpName, subnet.Status.U2OInterconnectionIP, "", subnet.Name, true); err != nil {
if _, _, _, err = c.ipam.GetStaticAddress(u2oInterconnName, u2oInterconnLrpName, subnet.Status.U2OInterconnectionIP, nil, subnet.Name, true); err != nil {
klog.Errorf("failed to init subnet u2o interonnection ip to ipam %v", subnet.Name, err)
}
}
Expand Down Expand Up @@ -345,7 +345,7 @@ func (c *Controller) InitIPAM() error {
} else {
ipamKey = fmt.Sprintf("node-%s", ip.Spec.PodName)
}
if _, _, _, err = c.ipam.GetStaticAddress(ipamKey, ip.Name, ip.Spec.IPAddress, ip.Spec.MacAddress, ip.Spec.Subnet, true); err != nil {
if _, _, _, err = c.ipam.GetStaticAddress(ipamKey, ip.Name, ip.Spec.IPAddress, &ip.Spec.MacAddress, ip.Spec.Subnet, true); err != nil {
klog.Errorf("failed to init IPAM from IP CR %s: %v", ip.Name, err)
}
}
Expand Down Expand Up @@ -376,7 +376,7 @@ func (c *Controller) InitIPAM() error {
ip := pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)]
mac := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)]
subnet := pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)]
_, _, _, err := c.ipam.GetStaticAddress(key, portName, ip, mac, subnet, true)
_, _, _, err := c.ipam.GetStaticAddress(key, portName, ip, &mac, subnet, true)
if err != nil {
klog.Errorf("failed to init pod %s.%s address %s: %v", podName, pod.Namespace, pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)], err)
} else {
Expand Down Expand Up @@ -421,7 +421,7 @@ func (c *Controller) InitIPAM() error {
} else {
ipamKey = vip.Name
}
if _, _, _, err = c.ipam.GetStaticAddress(ipamKey, vip.Name, vip.Status.V4ip, vip.Status.Mac, vip.Spec.Subnet, true); err != nil {
if _, _, _, err = c.ipam.GetStaticAddress(ipamKey, vip.Name, vip.Status.V4ip, &vip.Status.Mac, vip.Spec.Subnet, true); err != nil {
klog.Errorf("failed to init ipam from vip cr %s: %v", vip.Name, err)
}
}
Expand All @@ -433,7 +433,7 @@ func (c *Controller) InitIPAM() error {
}
for _, eip := range eips {
externalNetwork := util.GetExternalNetwork(eip.Spec.ExternalSubnet)
if _, _, _, err = c.ipam.GetStaticAddress(eip.Name, eip.Name, eip.Status.IP, eip.Spec.MacAddress, externalNetwork, true); err != nil {
if _, _, _, err = c.ipam.GetStaticAddress(eip.Name, eip.Name, eip.Status.IP, &eip.Spec.MacAddress, externalNetwork, true); err != nil {
klog.Errorf("failed to init ipam from iptables eip cr %s: %v", eip.Name, err)
}
}
Expand All @@ -443,7 +443,7 @@ func (c *Controller) InitIPAM() error {
return err
}
for _, oeip := range oeips {
if _, _, _, err = c.ipam.GetStaticAddress(oeip.Name, oeip.Name, oeip.Status.V4Ip, oeip.Status.MacAddress, oeip.Spec.ExternalSubnet, true); err != nil {
if _, _, _, err = c.ipam.GetStaticAddress(oeip.Name, oeip.Name, oeip.Status.V4Ip, &oeip.Status.MacAddress, oeip.Spec.ExternalSubnet, true); err != nil {
klog.Errorf("failed to init ipam from ovn eip cr %s: %v", oeip.Name, err)
}
}
Expand All @@ -455,9 +455,9 @@ func (c *Controller) InitIPAM() error {
for _, node := range nodes {
if node.Annotations[util.AllocatedAnnotation] == "true" {
portName := fmt.Sprintf("node-%s", node.Name)
mac := node.Annotations[util.MacAddressAnnotation]
v4IP, v6IP, _, err := c.ipam.GetStaticAddress(portName, portName,
node.Annotations[util.IpAddressAnnotation],
node.Annotations[util.MacAddressAnnotation],
node.Annotations[util.IpAddressAnnotation], &mac,
node.Annotations[util.LogicalSwitchAnnotation], true)
if err != nil {
klog.Errorf("failed to init node %s.%s address %s: %v", node.Name, node.Namespace, node.Annotations[util.IpAddressAnnotation], err)
Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,15 +242,15 @@ func (c *Controller) handleAddNode(key string) error {
var v4IP, v6IP, mac string
portName := fmt.Sprintf("node-%s", key)
if node.Annotations[util.AllocatedAnnotation] == "true" && node.Annotations[util.IpAddressAnnotation] != "" && node.Annotations[util.MacAddressAnnotation] != "" {
macStr := node.Annotations[util.MacAddressAnnotation]
v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(portName, portName, node.Annotations[util.IpAddressAnnotation],
node.Annotations[util.MacAddressAnnotation],
node.Annotations[util.LogicalSwitchAnnotation], true)
&macStr, node.Annotations[util.LogicalSwitchAnnotation], true)
if err != nil {
klog.Errorf("failed to alloc static ip addrs for node %v: %v", node.Name, err)
return err
}
} else {
v4IP, v6IP, mac, err = c.ipam.GetRandomAddress(portName, portName, "", c.config.NodeSwitch, nil, true)
v4IP, v6IP, mac, err = c.ipam.GetRandomAddress(portName, portName, nil, c.config.NodeSwitch, nil, true)
if err != nil {
klog.Errorf("failed to alloc random ip addrs for node %v: %v", node.Name, err)
return err
Expand Down
42 changes: 29 additions & 13 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -610,14 +610,23 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
}
ipStr := util.GetStringIP(v4IP, v6IP)
pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)] = ipStr
pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)] = mac
if mac == "" {
delete(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName))
} else {
pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)] = mac
}
pod.Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.CIDRBlock
pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.Gateway
pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)] = subnet.Name
pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true"
if pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)] == "" {
pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)] = c.config.PodNicType
if isOvnSubnet(podNet.Subnet) {
pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)] = subnet.Name
if pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)] == "" {
pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)] = c.config.PodNicType
}
} else {
delete(pod.Annotations, fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName))
delete(pod.Annotations, fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName))
}
pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true"
if isVmPod && c.config.EnableKeepVmIP {
pod.Annotations[fmt.Sprintf(util.VmTemplate, podNet.ProviderName)] = vmName
if err := c.changeVMSubnet(vmName, namespace, podNet.ProviderName, subnet.Name, pod); err != nil {
Expand Down Expand Up @@ -1422,11 +1431,18 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
return vip.Status.V4ip, vip.Status.V6ip, vip.Status.Mac, podNet.Subnet, nil
}

macStr := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)]
if macStr != "" {
if _, err := net.ParseMAC(macStr); err != nil {
return "", "", "", podNet.Subnet, err
var macStr *string
if isOvnSubnet(podNet.Subnet) {
mac := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)]
if mac != "" {
if _, err := net.ParseMAC(mac); err != nil {
return "", "", "", podNet.Subnet, err
}
macStr = &mac
}
} else {
macStr = new(string)
*macStr = ""
}

// Random allocate
Expand Down Expand Up @@ -1532,20 +1548,20 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
return "", "", "", podNet.Subnet, ipam.ErrNoAvailable
}

func (c *Controller) acquireStaticAddress(key, nicName, ip, mac, subnet string, liveMigration bool) (string, string, string, error) {
var v4IP, v6IP string
func (c *Controller) acquireStaticAddress(key, nicName, ip string, mac *string, subnet string, liveMigration bool) (string, string, string, error) {
var v4IP, v6IP, macStr string
var err error
for _, ipStr := range strings.Split(ip, ",") {
if net.ParseIP(ipStr) == nil {
return "", "", "", fmt.Errorf("failed to parse IP %s", ipStr)
}
}

if v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(key, nicName, ip, mac, subnet, !liveMigration); err != nil {
if v4IP, v6IP, macStr, err = c.ipam.GetStaticAddress(key, nicName, ip, mac, subnet, !liveMigration); err != nil {
klog.Errorf("failed to get static ip %v, mac %v, subnet %v, err %v", ip, mac, subnet, err)
return "", "", "", err
}
return v4IP, v6IP, mac, nil
return v4IP, v6IP, macStr, nil
}

func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind string) (bool, error) {
Expand Down
10 changes: 7 additions & 3 deletions pkg/controller/vip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ func (c *Controller) acquireStaticIpAddress(subnetName, name, nicName, ip string
}
}

if v4ip, v6ip, mac, err = c.ipam.GetStaticAddress(name, nicName, ip, mac, subnetName, checkConflict); err != nil {
if v4ip, v6ip, mac, err = c.ipam.GetStaticAddress(name, nicName, ip, nil, subnetName, checkConflict); err != nil {
klog.Errorf("failed to get static virtual ip '%s', mac '%s', subnet '%s', %v", ip, mac, subnetName, err)
return "", "", "", err
}
Expand All @@ -293,7 +293,7 @@ func (c *Controller) acquireIpAddress(subnetName, name, nicName string) (string,
checkConflict := true
var err error
for {
v4ip, v6ip, mac, err = c.ipam.GetRandomAddress(name, nicName, mac, subnetName, skippedAddrs, checkConflict)
v4ip, v6ip, mac, err = c.ipam.GetRandomAddress(name, nicName, nil, subnetName, skippedAddrs, checkConflict)
if err != nil {
return "", "", "", err
}
Expand Down Expand Up @@ -507,7 +507,11 @@ func (c *Controller) releaseVip(key string) error {
klog.Errorf("failed to patch label for vip '%s', %v", vip.Name, err)
return err
}
if _, _, _, err = c.ipam.GetStaticAddress(key, vip.Name, vip.Status.V4ip, vip.Status.Mac, vip.Spec.Subnet, false); err != nil {
mac := &vip.Status.Mac
if vip.Status.Mac == "" {
mac = nil
}
if _, _, _, err = c.ipam.GetStaticAddress(key, vip.Name, vip.Status.V4ip, mac, vip.Spec.Subnet, false); err != nil {
klog.Errorf("failed to recover IPAM from VIP CR %s: %v", vip.Name, err)
}
c.updateSubnetStatusQueue.Add(vip.Spec.Subnet)
Expand Down
4 changes: 2 additions & 2 deletions pkg/controller/vpc_nat_gw_eip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ func (c *Controller) acquireStaticEip(name, namespace, nicName, ip, externalSubn
}
}

if v4ip, v6ip, mac, err = c.ipam.GetStaticAddress(name, nicName, ip, mac, externalSubnet, checkConflict); err != nil {
if v4ip, v6ip, mac, err = c.ipam.GetStaticAddress(name, nicName, ip, nil, externalSubnet, checkConflict); err != nil {
klog.Errorf("failed to get static ip %v, mac %v, subnet %v, err %v", ip, mac, externalSubnet, err)
return "", "", "", err
}
Expand All @@ -579,7 +579,7 @@ func (c *Controller) acquireStaticEip(name, namespace, nicName, ip, externalSubn
func (c *Controller) acquireEip(name, namespace, nicName, externalSubnet string) (string, string, string, error) {
var skippedAddrs []string
for {
ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(name, nicName, "", externalSubnet, skippedAddrs, true)
ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(name, nicName, nil, externalSubnet, skippedAddrs, true)
if err != nil {
return "", "", "", err
}
Expand Down
21 changes: 13 additions & 8 deletions pkg/daemon/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned"
"github.com/kubeovn/kube-ovn/pkg/ovs"
"github.com/kubeovn/kube-ovn/pkg/request"
Expand All @@ -43,17 +44,17 @@ func createCniServerHandler(config *Configuration, controller *Controller) *cniS
return csh
}

func (csh cniServerHandler) providerExists(provider string) bool {
func (csh cniServerHandler) providerExists(provider string) (*kubeovnv1.Subnet, bool) {
if provider == "" || strings.HasSuffix(provider, util.OvnProvider) {
return true
return nil, true
}
subnets, _ := csh.Controller.subnetsLister.List(labels.Everything())
for _, subnet := range subnets {
if subnet.Spec.Provider == provider {
return true
return subnet.DeepCopy(), true
}
}
return false
return nil, false
}

func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Response) {
Expand All @@ -67,7 +68,8 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
return
}
klog.V(5).Infof("request body is %v", podRequest)
if exist := csh.providerExists(podRequest.Provider); !exist {
podSubnet, exist := csh.providerExists(podRequest.Provider)
if !exist {
errMsg := fmt.Errorf("provider %s not bind to any subnet", podRequest.Provider)
klog.Error(errMsg)
if err := resp.WriteHeaderAndEntity(http.StatusBadRequest, request.CniResponse{Err: errMsg.Error()}); err != nil {
Expand Down Expand Up @@ -115,7 +117,6 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
time.Sleep(1 * time.Second)
continue
}
macAddr = pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podRequest.Provider)]
ip = pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podRequest.Provider)]
cidr = pod.Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, podRequest.Provider)]
gw = pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, podRequest.Provider)]
Expand Down Expand Up @@ -189,7 +190,10 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
return
}

if err := csh.UpdateIPCr(podRequest, subnet, ip, macAddr); err != nil {
if subnet == "" && podSubnet != nil {
subnet = podSubnet.Name
}
if err := csh.UpdateIPCr(podRequest, subnet, ip); err != nil {
if err := resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: err.Error()}); err != nil {
klog.Errorf("failed to write response, %v", err)
}
Expand Down Expand Up @@ -275,6 +279,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
}

routes = append(podRequest.Routes, routes...)
macAddr = pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podRequest.Provider)]
klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, routes)
if nicType == util.InternalType {
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
Expand Down Expand Up @@ -324,7 +329,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
}
}

func (csh cniServerHandler) UpdateIPCr(podRequest request.CniRequest, subnet, ip, macAddr string) error {
func (csh cniServerHandler) UpdateIPCr(podRequest request.CniRequest, subnet, ip string) error {
ipCrName := ovs.PodNameToPortName(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider)
oriIpCr, err := csh.KubeOvnClient.KubeovnV1().IPs().Get(context.Background(), ipCrName, metav1.GetOptions{})
if err != nil {
Expand Down
Loading

0 comments on commit 4f015f6

Please sign in to comment.