Skip to content

Commit

Permalink
feat: suport vm live migration
Browse files Browse the repository at this point in the history
  • Loading branch information
@fanriming
fanriming committed Sep 1, 2021
1 parent 0c252cd commit 2439c86
Show file tree
Hide file tree
Showing 10 changed files with 156 additions and 109 deletions.
4 changes: 2 additions & 2 deletions pkg/controller/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ func (c *Controller) InitIPAM() error {
fmt.Sprintf("%s/%s", pod.Namespace, pod.Name),
pod.Annotations[util.IpAddressAnnotation],
pod.Annotations[util.MacAddressAnnotation],
pod.Annotations[util.LogicalSwitchAnnotation])
pod.Annotations[util.LogicalSwitchAnnotation], false)
if err != nil {
klog.Errorf("failed to init pod %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[util.IpAddressAnnotation], err)
}
Expand All @@ -309,7 +309,7 @@ func (c *Controller) InitIPAM() error {
portName := fmt.Sprintf("node-%s", node.Name)
v4IP, v6IP, _, err := c.ipam.GetStaticAddress(portName, node.Annotations[util.IpAddressAnnotation],
node.Annotations[util.MacAddressAnnotation],
node.Annotations[util.LogicalSwitchAnnotation])
node.Annotations[util.LogicalSwitchAnnotation], true)
if err != nil {
klog.Errorf("failed to init node %s.%s address %s: %v", node.Name, node.Namespace, node.Annotations[util.IpAddressAnnotation], err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ func (c *Controller) handleAddNode(key string) error {
if node.Annotations[util.AllocatedAnnotation] == "true" && node.Annotations[util.IpAddressAnnotation] != "" && node.Annotations[util.MacAddressAnnotation] != "" {
v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(portName, node.Annotations[util.IpAddressAnnotation],
node.Annotations[util.MacAddressAnnotation],
node.Annotations[util.LogicalSwitchAnnotation])
node.Annotations[util.LogicalSwitchAnnotation], true)
if err != nil {
klog.Errorf("failed to alloc static ip addrs for node %v: %v", node.Name, err)
return err
Expand Down
32 changes: 19 additions & 13 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -900,10 +900,11 @@ const (
)

type kubeovnNet struct {
Type providerType
ProviderName string
Subnet *kubeovnv1.Subnet
IsDefault bool
Type providerType
ProviderName string
Subnet *kubeovnv1.Subnet
IsDefault bool
AllowLiveMigration bool
}

func (c *Controller) getPodAttachmentNet(pod *v1.Pod) ([]*kubeovnNet, error) {
Expand Down Expand Up @@ -950,11 +951,15 @@ func (c *Controller) getPodAttachmentNet(pod *v1.Pod) ([]*kubeovnNet, error) {
// allocate kubeovn network
var providerName string
if util.IsOvnNetwork(netCfg) {
allowLiveMigration := false
isDefault := util.IsDefaultNet(pod.Annotations[util.DefaultNetworkAnnotation], attach)
if isDefault {
providerName = util.OvnProvider
} else {
providerName = fmt.Sprintf("%s.%s.ovn", attach.Name, attach.Namespace)
if pod.Annotations[fmt.Sprintf(util.LiveMigrationAnnotationTemplate, providerName)] == "true" {
allowLiveMigration = true
}
}
subnetName := pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, providerName)]
if subnetName == "" {
Expand All @@ -966,10 +971,11 @@ func (c *Controller) getPodAttachmentNet(pod *v1.Pod) ([]*kubeovnNet, error) {
return nil, err
}
result = append(result, &kubeovnNet{
Type: providerTypeOriginal,
ProviderName: providerName,
Subnet: subnet,
IsDefault: isDefault,
Type: providerTypeOriginal,
ProviderName: providerName,
Subnet: subnet,
IsDefault: isDefault,
AllowLiveMigration: allowLiveMigration,
})

}
Expand Down Expand Up @@ -1056,7 +1062,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
// Static allocate
if pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)] != "" {
ipStr := pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)]
return c.acquireStaticAddress(key, ipStr, macStr, podNet.Subnet.Name)
return c.acquireStaticAddress(key, ipStr, macStr, podNet.Subnet.Name, podNet.AllowLiveMigration)
}

// IPPool allocate
Expand All @@ -1071,7 +1077,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
klog.Errorf("static address %s for %s has been assigned", staticIP, key)
continue
}
if v4IP, v6IP, mac, err := c.acquireStaticAddress(key, staticIP, macStr, podNet.Subnet.Name); err == nil {
if v4IP, v6IP, mac, err := c.acquireStaticAddress(key, staticIP, macStr, podNet.Subnet.Name, podNet.AllowLiveMigration); err == nil {
return v4IP, v6IP, mac, nil
} else {
klog.Errorf("acquire address %s for %s failed, %v", staticIP, key, err)
Expand All @@ -1082,7 +1088,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
numStr := strings.Split(pod.Name, "-")[numIndex]
index, _ := strconv.Atoi(numStr)
if index < len(ipPool) {
return c.acquireStaticAddress(key, ipPool[index], macStr, podNet.Subnet.Name)
return c.acquireStaticAddress(key, ipPool[index], macStr, podNet.Subnet.Name, podNet.AllowLiveMigration)
}
}
klog.Errorf("alloc address for %s failed, return NoAvailableAddress", key)
Expand All @@ -1101,7 +1107,7 @@ func generatePatchPayload(annotations map[string]string, op string) []byte {
return []byte(fmt.Sprintf(patchPayloadTemplate, op, raw))
}

func (c *Controller) acquireStaticAddress(key, ip, mac, subnet string) (string, string, string, error) {
func (c *Controller) acquireStaticAddress(key, ip, mac, subnet string, liveMigration bool) (string, string, string, error) {
var v4IP, v6IP string
var err error
for _, ipStr := range strings.Split(ip, ",") {
Expand All @@ -1110,7 +1116,7 @@ func (c *Controller) acquireStaticAddress(key, ip, mac, subnet string) (string,
}
}

if v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(key, ip, mac, subnet); err != nil {
if v4IP, v6IP, mac, err = c.ipam.GetStaticAddress(key, ip, mac, subnet, !liveMigration); err != nil {
klog.Errorf("failed to get static ip %v, mac %v, subnet %v, err %v", ip, mac, subnet, err)
return "", "", "", err
}
Expand Down
16 changes: 14 additions & 2 deletions pkg/daemon/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon

klog.Infof("add port request %v", podRequest)
var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, providerNetwork, ifName, nicType, netns, podNicName string
var isDefaultRoute bool
var pod *v1.Pod
var err error
for i := 0; i < 15; i++ {
Expand Down Expand Up @@ -112,6 +113,17 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
nicType = pod.Annotations[util.PodNicAnnotation]
}
netns = podRequest.NetNs

switch pod.Annotations[fmt.Sprintf(util.DefaultRouteAnnotationTemplate, podRequest.Provider)] {
case "true":
isDefaultRoute = true
case "false":
isDefaultRoute = false
default:
if ifName == "" || ifName == "eth0" {
isDefaultRoute = true
}
}
break
}

Expand Down Expand Up @@ -173,10 +185,10 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon

klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, podRequest.Routes)
if nicType == util.InternalType {
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, podRequest.Routes, ingress, egress, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, ingress, egress, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
} else {
podNicName = ifName
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, podRequest.Routes, ingress, egress, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, ingress, egress, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
}
if err != nil {
errMsg := fmt.Errorf("configure nic failed %v", err)
Expand Down
20 changes: 10 additions & 10 deletions pkg/daemon/ovs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/kubeovn/kube-ovn/pkg/util"
)

func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, routes []request.Route, ingress, egress, DeviceID, nicType, podNetns string, checkGw bool) error {
func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, ingress, egress, DeviceID, nicType, podNetns string, checkGw bool) error {
var err error
var hostNicName, containerNicName string
if DeviceID == "" {
Expand Down Expand Up @@ -75,7 +75,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns,
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", netns, err)
}
if err = configureContainerNic(containerNicName, ifName, ip, gateway, routes, macAddr, podNS, mtu, nicType, checkGw); err != nil {
if err = configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, routes, macAddr, podNS, mtu, nicType, checkGw); err != nil {
return err
}
return nil
Expand Down Expand Up @@ -152,7 +152,7 @@ func configureHostNic(nicName string) error {
return nil
}

func configureContainerNic(nicName, ifName string, ipAddr, gateway string, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, checkGw bool) error {
func configureContainerNic(nicName, ifName string, ipAddr, gateway string, isDefaultRoute bool, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, checkGw bool) error {
containerLink, err := netlink.LinkByName(nicName)
if err != nil {
return fmt.Errorf("can not find container nic %s: %v", nicName, err)
Expand Down Expand Up @@ -201,20 +201,20 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, route
}
}

if ifName == "eth0" {
if isDefaultRoute {
// Only eth0 requires the default route and gateway
switch util.CheckProtocol(ipAddr) {
case kubeovnv1.ProtocolIPv4:
_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")
err = netlink.RouteAdd(&netlink.Route{
err = netlink.RouteReplace(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gateway),
})
case kubeovnv1.ProtocolIPv6:
_, defaultNet, _ := net.ParseCIDR("::/0")
err = netlink.RouteAdd(&netlink.Route{
err = netlink.RouteReplace(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Expand All @@ -223,7 +223,7 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, route
case kubeovnv1.ProtocolDual:
gws := strings.Split(gateway, ",")
_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")
err = netlink.RouteAdd(&netlink.Route{
err = netlink.RouteReplace(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Expand All @@ -234,7 +234,7 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, route
}

_, defaultNet, _ = net.ParseCIDR("::/0")
err = netlink.RouteAdd(&netlink.Route{
err = netlink.RouteReplace(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Expand Down Expand Up @@ -850,7 +850,7 @@ func renameLink(curName, newName string) error {
return nil
}

func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, routes []request.Route, ingress, egress, DeviceID, nicType, podNetns string, checkGw bool) (string, error) {
func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, ingress, egress, DeviceID, nicType, podNetns string, checkGw bool) (string, error) {
_, containerNicName := generateNicName(containerID, ifName)
ipStr := util.GetIpWithoutMask(ip)
ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)
Expand Down Expand Up @@ -882,7 +882,7 @@ func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace,
if err != nil {
return containerNicName, fmt.Errorf("failed to open netns %q: %v", netns, err)
}
if err = configureContainerNic(containerNicName, ifName, ip, gateway, routes, macAddr, podNS, mtu, nicType, checkGw); err != nil {
if err = configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, routes, macAddr, podNS, mtu, nicType, checkGw); err != nil {
return containerNicName, err
}
return containerNicName, nil
Expand Down
8 changes: 4 additions & 4 deletions pkg/ipam/ipam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func (ipam *IPAM) GetRandomAddress(podName, subnetName string, skippedAddrs []st
return string(v4IP), string(v6IP), mac, err
}

func (ipam *IPAM) GetStaticAddress(podName, ip, mac, subnetName string) (string, string, string, error) {
func (ipam *IPAM) GetStaticAddress(podName, ip, mac, subnetName string, checkConflict bool) (string, string, string, error) {
ipam.mutex.RLock()
defer ipam.mutex.RUnlock()
if subnet, ok := ipam.Subnets[subnetName]; !ok {
Expand All @@ -60,7 +60,7 @@ func (ipam *IPAM) GetStaticAddress(podName, ip, mac, subnetName string) (string,
var err error
var ipAddr IP
for _, ipStr := range strings.Split(ip, ",") {
ipAddr, mac, err = subnet.GetStaticAddress(podName, IP(ipStr), mac, false)
ipAddr, mac, err = subnet.GetStaticAddress(podName, IP(ipStr), mac, false, checkConflict)
if err != nil {
return "", "", "", err
}
Expand Down Expand Up @@ -158,7 +158,7 @@ func (ipam *IPAM) AddOrUpdateSubnet(name, cidrStr string, excludeIps []string) e
subnet.joinFreeWithReserve()
for podName, ip := range subnet.V4PodToIP {
mac := subnet.PodToMac[podName]
if _, _, err := subnet.GetStaticAddress(podName, ip, mac, true); err != nil {
if _, _, err := subnet.GetStaticAddress(podName, ip, mac, true, true); err != nil {
klog.Errorf("%s address not in subnet %s new cidr %s", podName, name, cidrStr)
}
}
Expand All @@ -173,7 +173,7 @@ func (ipam *IPAM) AddOrUpdateSubnet(name, cidrStr string, excludeIps []string) e
subnet.joinFreeWithReserve()
for podName, ip := range subnet.V6PodToIP {
mac := subnet.PodToMac[podName]
if _, _, err := subnet.GetStaticAddress(podName, ip, mac, true); err != nil {
if _, _, err := subnet.GetStaticAddress(podName, ip, mac, true, true); err != nil {
klog.Errorf("%s address not in subnet %s new cidr %s", podName, name, cidrStr)
}
}
Expand Down

0 comments on commit 2439c86

Please sign in to comment.