feat: add cni side logical to support ipam for multi-nic

Makefile

@@ -68,6 +68,7 @@ kind-init-ha:
 	kind delete cluster --name=kube-ovn
 	kind create cluster --config yamls/kind.yaml --name kube-ovn
 	kind load docker-image --name kube-ovn ${REGISTRY}/kube-ovn:${RELEASE_TAG}
+	kind load docker-image --name kube-ovn nfvpe/multus:v3.4
 	bash dist/images/install.sh
 
 kind-reload:
@@ -81,6 +82,8 @@ uninstall:
 	bash dist/images/cleanup.sh
 
 e2e:
+	docker pull index.alauda.cn/claas/pause:3.1
+	kind load docker-image --name kube-ovn index.alauda.cn/claas/pause:3.1
 	ginkgo -p --slowSpecThreshold=60 test/e2e
 
 ut:

cmd/cni/cni.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/alauda/kube-ovn/pkg/util"
 	"net"
 	"runtime"
 	"strings"
@@ -49,7 +50,9 @@ func cmdAdd(args *skel.CmdArgs) error {
 		PodName:      podName,
 		PodNamespace: podNamespace,
 		ContainerID:  args.ContainerID,
-		NetNs:        args.Netns})
+		NetNs:        args.Netns,
+		Provider:     netConf.Provider,
+	})
 	if err != nil {
 		return err
 	}
@@ -112,22 +115,42 @@ func cmdDel(args *skel.CmdArgs) error {
 		PodName:      podName,
 		PodNamespace: podNamespace,
 		ContainerID:  args.ContainerID,
-		NetNs:        args.Netns})
+		NetNs:        args.Netns,
+		Provider:     netConf.Provider,
+	})
+}
+
+type ipamConf struct {
+	ServerSocket string `json:"server_socket"`
+	Provider     string `json:"provider"`
 }
 
 type netConf struct {
 	types.NetConf
-	ServerSocket string `json:"server_socket"`
+	ServerSocket string    `json:"server_socket"`
+	Provider     string    `json:"provider"`
+	IPAM         *ipamConf `json:"ipam"`
 }
 
 func loadNetConf(bytes []byte) (*netConf, string, error) {
 	n := &netConf{}
 	if err := json.Unmarshal(bytes, n); err != nil {
 		return nil, "", fmt.Errorf("failed to load netconf: %v", err)
 	}
+
+	if n.IPAM != nil {
+		n.Provider = n.IPAM.Provider
+		n.ServerSocket = n.IPAM.ServerSocket
+	}
+
 	if n.ServerSocket == "" {
-		return nil, "", fmt.Errorf("server_socket is required in cni.conf")
+		return nil, "", fmt.Errorf("server_socket is required in cni.conf, %+v", n)
 	}
+
+	if n.Provider == "" {
+		n.Provider = util.OvnProvider
+	}
+
 	return n, n.CNIVersion, nil
 }
 

dist/images/Dockerfile

@@ -38,7 +38,7 @@ RUN curl -L https://storage.googleapis.com/kubernetes-release/release/${KUBE_VER
  && chmod +x /usr/bin/kubectl
 
 COPY *.sh /kube-ovn/
-COPY 00-kube-ovn.conflist /kube-ovn/00-kube-ovn.conflist
+COPY 01-kube-ovn.conflist /kube-ovn/01-kube-ovn.conflist
 
 WORKDIR /kube-ovn
 

dist/images/install-cni.sh

@@ -26,8 +26,8 @@ exit_with_error(){
 CNI_BIN_SRC=/kube-ovn/kube-ovn
 CNI_BIN_DST=/opt/cni/bin/kube-ovn
 
-CNI_CONF_SRC=/kube-ovn/00-kube-ovn.conflist
-CNI_CONF_DST=/etc/cni/net.d/00-kube-ovn.conflist
+CNI_CONF_SRC=/kube-ovn/01-kube-ovn.conflist
+CNI_CONF_DST=/etc/cni/net.d/01-kube-ovn.conflist
 
 LOOPBACK_BIN_SRC=/loopback
 LOOPBACK_BIN_DST=/opt/cni/bin/loopback

dist/images/uninstall.sh

@@ -8,4 +8,4 @@ rm -rf /etc/openvswitch/*
 rm -rf /etc/ovn/*
 rm -rf /var/log/openvswitch/*
 rm -rf /var/log/ovn/*
-rm -rf /etc/cni/net.d/00-kube-ovn.conflist
+rm -rf /etc/cni/net.d/01-kube-ovn.conflist

pkg/controller/pod.go

@@ -526,7 +526,7 @@ func (c *Controller) getPodAttachmentSubnet(pod *v1.Pod) ([]*kubeovnv1.Subnet, e
 }
 
 func (c *Controller) acquireAddress(pod *v1.Pod, subnet *kubeovnv1.Subnet) (string, string, error) {
-	key := fmt.Sprintf("%s/%s", pod.Name, pod.Namespace)
+	key := fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)
 
 	// Random allocate
 	if pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, subnet.Spec.Provider)] == "" &&

pkg/daemon/handler.go

@@ -50,7 +50,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 			return
 		}
 
-		if pod.Annotations[util.AllocatedAnnotation] != "true" {
+		if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podRequest.Provider)] != "true" {
 			klog.Infof("wait address for  pod %s/%s ", podRequest.PodNamespace, podRequest.PodName)
 			// wait controller assign an address
 			time.Sleep(1 * time.Second)
@@ -63,11 +63,11 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 			time.Sleep(1 * time.Second)
 			continue
 		}
-		macAddr = pod.Annotations[util.MacAddressAnnotation]
-		ip = pod.Annotations[util.IpAddressAnnotation]
-		cidr = pod.Annotations[util.CidrAnnotation]
-		gw = pod.Annotations[util.GatewayAnnotation]
-		subnet = pod.Annotations[util.LogicalSwitchAnnotation]
+		macAddr = pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podRequest.Provider)]
+		ip = pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podRequest.Provider)]
+		cidr = pod.Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, podRequest.Provider)]
+		gw = pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, podRequest.Provider)]
+		subnet = pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podRequest.Provider)]
 		ingress = pod.Annotations[util.IngressRateAnnotation]
 		egress = pod.Annotations[util.EgressRateAnnotation]
 		break
@@ -137,13 +137,15 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 	}
 
 	ipAddr = fmt.Sprintf("%s/%s", ip, strings.Split(cidr, "/")[1])
-	klog.Infof("create container mac %s, ip %s, cidr %s, gw %s", macAddr, ipAddr, cidr, gw)
-	err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.NetNs, podRequest.ContainerID, macAddr, ipAddr, gw, ingress, egress)
-	if err != nil {
-		errMsg := fmt.Errorf("configure nic failed %v", err)
-		klog.Error(errMsg)
-		resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()})
-		return
+	if podRequest.Provider == util.OvnProvider {
+		klog.Infof("create container mac %s, ip %s, cidr %s, gw %s", macAddr, ipAddr, cidr, gw)
+		err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.NetNs, podRequest.ContainerID, macAddr, ipAddr, gw, ingress, egress)
+		if err != nil {
+			errMsg := fmt.Errorf("configure nic failed %v", err)
+			klog.Error(errMsg)
+			resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()})
+			return
+		}
 	}
 	resp.WriteHeaderAndEntity(http.StatusOK, request.CniResponse{Protocol: util.CheckProtocol(ipAddr), IpAddress: strings.Split(ipAddr, "/")[0], MacAddress: macAddr, CIDR: cidr, Gateway: gw})
 }
@@ -157,14 +159,18 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon
 		resp.WriteHeaderAndEntity(http.StatusBadRequest, request.CniResponse{Err: errMsg.Error()})
 		return
 	}
+
 	klog.Infof("delete port request %v", podRequest)
-	err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID)
-	if err != nil {
-		errMsg := fmt.Errorf("del nic failed %v", err)
-		klog.Error(errMsg)
-		resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()})
-		return
+	if podRequest.Provider == util.OvnProvider {
+		err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID)
+		if err != nil {
+			errMsg := fmt.Errorf("del nic failed %v", err)
+			klog.Error(errMsg)
+			resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()})
+			return
+		}
 	}
+
 	err = csh.KubeOvnClient.KubeovnV1().IPs().Delete(fmt.Sprintf("%s.%s", podRequest.PodName, podRequest.PodNamespace), &metav1.DeleteOptions{})
 	if err != nil && !k8serrors.IsNotFound(err) {
 		errMsg := fmt.Errorf("del ipcrd for %s failed %v", fmt.Sprintf("%s.%s", podRequest.PodName, podRequest.PodNamespace), err)

pkg/ipam/ipam.go

@@ -56,7 +56,9 @@ func (ipam *IPAM) ReleaseAddressByPod(podName string) {
 	defer ipam.mutex.RUnlock()
 	for _, subnet := range ipam.Subnets {
 		ip, mac := subnet.ReleaseAddress(podName)
-		klog.Infof("release %s %s for %s", ip, mac, podName)
+		if ip != "" {
+			klog.Infof("release %s %s for %s", ip, mac, podName)
+		}
 	}
 	return
 }
@@ -89,13 +91,15 @@ func (ipam *IPAM) AddOrUpdateSubnet(name, cidrStr string, excludeIps []string) e
 	if err != nil {
 		return err
 	}
+	klog.Infof("adding new subnet %s", name)
 	ipam.Subnets[name] = subnet
 	return nil
 }
 
 func (ipam *IPAM) DeleteSubnet(subnetName string) {
 	ipam.mutex.Lock()
 	defer ipam.mutex.Unlock()
+	klog.Infof("delete subnet %s", subnetName)
 	delete(ipam.Subnets, subnetName)
 }
 

pkg/request/cniserver.go

@@ -20,6 +20,7 @@ type CniRequest struct {
 	PodNamespace string `json:"pod_namespace"`
 	ContainerID  string `json:"container_id"`
 	NetNs        string `json:"net_ns"`
+	Provider     string `json:"provider"`
 }
 
 // CniResponse is the cniserver response format

test/e2e/ip/static_ip.go

@@ -11,6 +11,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"os"
+	"time"
 )
 
 var _ = Describe("[IP Allocation]", func() {
@@ -53,6 +54,7 @@ var _ = Describe("[IP Allocation]", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(pod.Annotations[util.AllocatedAnnotation]).To(Equal("true"))
 
+			time.Sleep(1 * time.Second)
 			ip, err := f.OvnClientSet.KubeovnV1().IPs().Get(fmt.Sprintf("%s.%s", name, namespace), metav1.GetOptions{})
 			Expect(err).NotTo(HaveOccurred())
 			Expect(ip.Spec.IPAddress).To(Equal("12.10.0.10"))