refactor ipam

Use kube-ovn ipam to replace ovn dynamic address allocation. We will design more flexible ip allocation strategy and provide ipam capacity to other cni plugin later.

Makefile

@@ -5,7 +5,7 @@ REGISTRY=index.alauda.cn/alaudak8s
 DEV_TAG=dev
 RELEASE_TAG=$(shell cat VERSION)
 
-.PHONY: build-dev-images build-go build-bin lint up down halt suspend resume kind push-dev push-release
+.PHONY: build-dev-images build-go build-bin lint up down halt suspend resume kind-init kind-init-ha kind-reload push-dev push-release e2e ut
 
 build-dev-images: build-bin
 	docker build -t ${REGISTRY}/kube-ovn:${DEV_TAG} -f dist/images/Dockerfile dist/images/
@@ -74,11 +74,14 @@ kind-init-ha:
 	kubectl apply -f yamls/kube-ovn.yaml
 
 kind-reload:
-    kind load docker-image --name kube-ovn ${REGISTRY}/kube-ovn:${RELEASE_TAG}
-	kubectl delete pod -n kube-ovn --all
+	kind load docker-image --name kube-ovn ${REGISTRY}/kube-ovn:${RELEASE_TAG}
+	kubectl delete pod -n kube-system -l app=kube-ovn-controller
 
 kind-clean:
 	kind delete cluster --name=kube-ovn
 
 e2e:
 	ginkgo -p --slowSpecThreshold=60 test/e2e
+
+ut:
+	ginkgo -p --slowSpecThreshold=60 test/unittest

pkg/controller/controller.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	ovnipam "github.com/alauda/kube-ovn/pkg/ipam"
 	"time"
 
 	kubeovnv1 "github.com/alauda/kube-ovn/pkg/apis/kubeovn/v1"
@@ -31,6 +32,7 @@ const controllerAgentName = "ovn-controller"
 type Controller struct {
 	config    *Configuration
 	ovnClient *ovs.Client
+	ipam      *ovnipam.IPAM
 
 	podsLister v1.PodLister
 	podsSynced cache.InformerSynced
@@ -112,6 +114,7 @@ func NewController(config *Configuration) *Controller {
 	controller := &Controller{
 		config:    config,
 		ovnClient: ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.OvnNbTimeout, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
+		ipam:      ovnipam.NewIPAM(),
 
 		subnetsLister:           subnetInformer.Lister(),
 		subnetSynced:            subnetInformer.Informer().HasSynced,
@@ -225,6 +228,10 @@ func (c *Controller) Run(stopCh <-chan struct{}) {
 		klog.Fatalf("failed to init ovn resource %v", err)
 	}
 
+	if err := c.InitIPAM(); err != nil {
+		klog.Fatalf("failed to init ipam %v", err)
+	}
+
 	c.informerFactory.Start(stopCh)
 	c.kubeovnInformerFactory.Start(stopCh)
 

pkg/controller/init.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"fmt"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
 	"strings"
 
 	kubeovnv1 "github.com/alauda/kube-ovn/pkg/apis/kubeovn/v1"
@@ -136,3 +137,54 @@ func (c *Controller) initLoadBalancer() error {
 	}
 	return nil
 }
+
+func (c *Controller) InitIPAM() error {
+	subnets, err := c.subnetsLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list subnet, %v", err)
+		return err
+	}
+	for _, subnet := range subnets {
+		if err := c.ipam.AddOrUpdateSubnet(subnet.Name, subnet.Spec.CIDRBlock, subnet.Spec.ExcludeIps); err != nil {
+			klog.Errorf("failed to init subnet %s, %v", subnet.Name, err)
+		}
+	}
+
+	pods, err := c.podsLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list pods, %v", err)
+		return err
+	}
+	for _, pod := range pods {
+		if pod.Annotations[util.AllocatedAnnotation] == "true" &&
+			pod.Annotations[util.LogicalSwitchAnnotation] != "" {
+			_, _, err := c.ipam.GetStaticAddress(
+				fmt.Sprintf("%s/%s", pod.Namespace, pod.Name),
+				pod.Annotations[util.IpAddressAnnotation],
+				pod.Annotations[util.MacAddressAnnotation],
+				pod.Annotations[util.LogicalSwitchAnnotation])
+			if err != nil {
+				klog.Errorf("failed to init pod %s.%s address %s, %v", pod.Name, pod.Namespace, pod.Annotations[util.IpAddressAnnotation], err)
+			}
+		}
+	}
+
+	nodes, err := c.nodesLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list nodes, %v", err)
+		return err
+	}
+	for _, node := range nodes {
+		if node.Annotations[util.AllocatedAnnotation] == "true" {
+			portName := fmt.Sprintf("node-%s", node.Name)
+			_, _, err := c.ipam.GetStaticAddress(portName, node.Annotations[util.IpAddressAnnotation],
+				node.Annotations[util.MacAddressAnnotation],
+				node.Annotations[util.LogicalSwitchAnnotation])
+			if err != nil {
+				klog.Errorf("failed to init node %s.%s address %s, %v", node.Name, node.Namespace, node.Annotations[util.IpAddressAnnotation], err)
+			}
+		}
+	}
+
+	return nil
+}

pkg/controller/node.go

@@ -192,42 +192,48 @@ func (c *Controller) handleAddNode(key string) error {
 		return err
 	}
 
-	nic, err := c.ovnClient.CreatePort(
-		c.config.NodeSwitch, fmt.Sprintf("node-%s", key),
-		node.Annotations[util.IpAddressAnnotation],
-		node.Annotations[util.CidrAnnotation],
-		node.Annotations[util.MacAddressAnnotation])
+	subnet, err := c.subnetsLister.Get(c.config.NodeSwitch)
 	if err != nil {
+		klog.Errorf("failed to get node subnet %v", err)
 		return err
 	}
 
-	nodeAddr := getNodeInternalIP(node)
-	if util.CheckProtocol(nodeAddr) == util.CheckProtocol(nic.IpAddress) {
-		err = c.ovnClient.AddStaticRoute("", nodeAddr, strings.Split(nic.IpAddress, "/")[0], c.config.ClusterRouter)
+	var ip, mac string
+	portName := fmt.Sprintf("node-%s", key)
+	if node.Annotations[util.AllocatedAnnotation] == "true" {
+		return nil
+	} else {
+		ip, mac, err = c.ipam.GetRandomAddress(portName, c.config.NodeSwitch)
 		if err != nil {
-			klog.Errorf("failed to add static router from node to ovn0 %v", err)
 			return err
 		}
 	}
-
-	subnet, err := c.subnetsLister.Get(c.config.NodeSwitch)
-	if err != nil {
-		klog.Errorf("failed to get node subnet %v", err)
+	if err := c.ovnClient.CreatePort(c.config.NodeSwitch, portName, ip, subnet.Spec.CIDRBlock, mac); err != nil {
 		return err
 	}
 
+	nodeAddr := getNodeInternalIP(node)
+	if util.CheckProtocol(nodeAddr) == util.CheckProtocol(ip) {
+		err = c.ovnClient.AddStaticRoute("", nodeAddr, strings.Split(ip, "/")[0], c.config.ClusterRouter)
+		if err != nil {
+			klog.Errorf("failed to add static router from node to ovn0 %v", err)
+			return err
+		}
+	}
+
 	patchPayloadTemplate :=
 		`[{
         "op": "%s",
         "path": "/metadata/annotations",
         "value": %s
     }]`
 	payload := map[string]string{
-		util.IpAddressAnnotation:     nic.IpAddress,
-		util.MacAddressAnnotation:    nic.MacAddress,
+		util.IpAddressAnnotation:     ip,
+		util.MacAddressAnnotation:    mac,
 		util.CidrAnnotation:          subnet.Spec.CIDRBlock,
 		util.GatewayAnnotation:       subnet.Spec.Gateway,
 		util.LogicalSwitchAnnotation: c.config.NodeSwitch,
+		util.AllocatedAnnotation:     "true",
 		util.PortNameAnnotation:      fmt.Sprintf("node-%s", key),
 	}
 	raw, _ := json.Marshal(payload)
@@ -256,17 +262,17 @@ func (c *Controller) handleAddNode(key string) error {
 					PodName:    key,
 					Subnet:     c.config.NodeSwitch,
 					NodeName:   key,
-					IPAddress:  nic.IpAddress,
-					MacAddress: nic.MacAddress,
+					IPAddress:  ip,
+					MacAddress: mac,
 				},
 			})
 			if err != nil {
-				errMsg := fmt.Errorf("failed to create ip crd for %s, %v", nic.IpAddress, err)
+				errMsg := fmt.Errorf("failed to create ip crd for %s, %v", ip, err)
 				klog.Error(errMsg)
 				return errMsg
 			}
 		} else {
-			errMsg := fmt.Errorf("failed to get ip crd for %s, %v", nic.IpAddress, err)
+			errMsg := fmt.Errorf("failed to get ip crd for %s, %v", ip, err)
 			klog.Error(errMsg)
 			return errMsg
 		}
@@ -282,12 +288,12 @@ func (c *Controller) handleAddNode(key string) error {
 		ipCr.Spec.Namespace = ""
 		ipCr.Spec.Subnet = c.config.NodeSwitch
 		ipCr.Spec.NodeName = key
-		ipCr.Spec.IPAddress = nic.IpAddress
-		ipCr.Spec.MacAddress = nic.MacAddress
+		ipCr.Spec.IPAddress = ip
+		ipCr.Spec.MacAddress = mac
 		ipCr.Spec.ContainerID = ""
 		_, err := c.config.KubeOvnClient.KubeovnV1().IPs().Update(ipCr)
 		if err != nil {
-			errMsg := fmt.Errorf("failed to create ip crd for %s, %v", nic.IpAddress, err)
+			errMsg := fmt.Errorf("failed to create ip crd for %s, %v", ip, err)
 			klog.Error(errMsg)
 			return errMsg
 		}
@@ -298,31 +304,23 @@ func (c *Controller) handleAddNode(key string) error {
 
 func (c *Controller) handleDeleteNode(key string) error {
 	portName := fmt.Sprintf("node-%s", key)
-	err := c.ovnClient.DeletePort(portName)
-	if err != nil {
+	if err := c.ovnClient.DeletePort(portName); err != nil {
 		klog.Errorf("failed to delete node switch port node-%s %v", key, err)
 		return err
 	}
 
-	ipCr, err := c.config.KubeOvnClient.KubeovnV1().IPs().Get(portName, metav1.GetOptions{})
-	if err != nil {
-		if k8serrors.IsNotFound(err) {
-			return nil
-		}
-		return err
-	}
-
-	if err := c.ovnClient.DeleteStaticRouteByNextHop(ipCr.Spec.IPAddress); err != nil {
+	if err := c.config.KubeOvnClient.KubeovnV1().IPs().Delete(portName, &metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
 		return err
 	}
 
-	err = c.config.KubeOvnClient.KubeovnV1().IPs().Delete(portName, &metav1.DeleteOptions{})
-	if err != nil {
-		if k8serrors.IsNotFound(err) {
-			return nil
+	ip, _, exist := c.ipam.GetPodAddress(portName)
+	if exist {
+		if err := c.ovnClient.DeleteStaticRouteByNextHop(ip); err != nil {
+			return err
 		}
-		return err
 	}
+
+	c.ipam.ReleaseAddressByPod(portName)
 	return nil
 }