add loadbalancer service (#1611)

.github/workflows/build-x86-image.yaml

@@ -1057,6 +1057,15 @@ jobs:
         run: |
           docker load --input kube-ovn.tar
 
+      - name: Download image
+        uses: actions/download-artifact@v3
+        with:
+          name: vpc-nat-gateway
+
+      - name: Load Image
+        run: |
+          docker load --input vpc-nat-gateway.tar
+
       - name: Install Kube-OVN & Multus
         uses: nick-invision/retry@v2
         with:
@@ -1078,7 +1087,7 @@ jobs:
           go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
-          make e2e
+          make e2e-multus
 
   ovnic-e2e:
     needs: build

Makefile

@@ -14,6 +14,8 @@ MULTUS_YAML = https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/
 CILIUM_VERSION = 1.10.9
 CILIUM_IMAGE_REPO = quay.io/cilium/cilium
 
+VPC_NAT_GW_IMG = $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG)
+
 # ARCH could be amd64,arm64
 ARCH = amd64
 
@@ -298,11 +300,14 @@ kind-install-multus:
 	if ! docker images --format "{{.Repository}}:{{.Tag}}" | grep -qw "^$(MULTUS_IMAGE)$$"; then \
 		docker pull "$(MULTUS_IMAGE)"; \
 	fi
+
 	kind load docker-image --name kube-ovn "$(MULTUS_IMAGE)"
 	kubectl apply -f "$(MULTUS_YAML)"
 	kubectl -n kube-system rollout status ds kube-multus-ds
+	kubectl apply -f yamls/lb-svc-attachment.yaml
 	kind load docker-image --name kube-ovn $(REGISTRY)/kube-ovn:$(RELEASE_TAG)
-	ENABLE_SSL=true CNI_CONFIG_PRIORITY=10 dist/images/install.sh
+	kind load docker-image --name kube-ovn $(VPC_NAT_GW_IMG)
+	ENABLE_SSL=true ENABLE_LB_SVC=true CNI_CONFIG_PRIORITY=10 dist/images/install.sh
 	kubectl describe no
 
 .PHONY: kind-install-ic
@@ -460,6 +465,10 @@ e2e-ovn-ebpf:
 	docker run -d --name kube-ovn-e2e --network kind --cap-add=NET_ADMIN $(REGISTRY)/kube-ovn:$(RELEASE_TAG) sleep infinity
 	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e-ebpf
 
+.PHONY: e2e-multus
+e2e-multus:
+	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e-multus
+
 .PHONY: clean
 clean:
 	$(RM) dist/images/kube-ovn dist/images/kube-ovn-cmd

charts/templates/ovn-CR.yaml

@@ -51,6 +51,7 @@ rules:
     resources:
       - networkpolicies
       - services
+      - services/status
       - endpoints
       - statefulsets
       - daemonsets

dist/images/install.sh

@@ -17,6 +17,7 @@ LS_DNAT_MOD_DL_DST=${LS_DNAT_MOD_DL_DST:-true}
 WITHOUT_KUBE_PROXY=${WITHOUT_KUBE_PROXY:-false}
 ENABLE_EXTERNAL_VPC=${ENABLE_EXTERNAL_VPC:-true}
 CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01}
+ENABLE_LB_SVC=${ENABLE_LB_SVC:-false}
 # The nic to support container network can be a nic name or a group of regex
 # separated by comma, if empty will use the nic that the default route use
 IFACE=${IFACE:-}
@@ -1444,6 +1445,7 @@ rules:
     resources:
       - networkpolicies
       - services
+      - services/status
       - endpoints
       - statefulsets
       - daemonsets
@@ -1929,6 +1931,7 @@ rules:
     resources:
       - networkpolicies
       - services
+      - services/status
       - endpoints
       - statefulsets
       - daemonsets
@@ -2577,6 +2580,7 @@ spec:
           - --inspect-interval=$INSPECT_INTERVAL
           - --log_file=/var/log/kube-ovn/kube-ovn-controller.log
           - --log_file_max_size=0
+          - --enable-lb-svc=$ENABLE_LB_SVC
           env:
             - name: ENABLE_SSL
               value: "$ENABLE_SSL"

dist/images/vpcnatgateway/Dockerfile

@@ -12,3 +12,4 @@ RUN set -ex \
 
 WORKDIR /kube-ovn
 COPY nat-gateway.sh /kube-ovn/
+COPY lb-svc.sh /kube-ovn/

dist/images/vpcnatgateway/lb-svc.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+ROUTE_TABLE=100
+
+function exec_cmd() {
+    cmd=${@:1:${#}}
+    $cmd
+    ret=$?
+    if [ $ret -ne 0 ]; then
+        echo "failed to exec \"$cmd\""
+        exit $ret
+    fi
+}
+
+function init() {
+    ip link set net1 up
+    ip link set dev net1 arp off
+
+    if [ $(ip rule show iif net1 | wc -l) -eq 0 ]; then
+        exec_cmd "ip rule add iif net1 table $ROUTE_TABLE"
+    fi
+    if [ $(ip rule show iif eth0 | wc -l) -eq 0 ]; then
+        exec_cmd "ip rule add iif eth0 table $ROUTE_TABLE"
+    fi
+}
+
+function add_eip() {
+    for rule in $@
+    do
+        arr=(${rule//,/ })
+        eip=${arr[0]}
+        eip_without_prefix=(${eip//\// })
+        eip_network=$(ipcalc -n $eip | awk -F '=' '{print $2}')
+        eip_prefix=$(ipcalc -p $eip | awk -F '=' '{print $2}')
+        gateway=${arr[1]}
+
+        exec_cmd "ip addr replace $eip dev net1"
+        exec_cmd "ip route replace $eip_network/$eip_prefix dev net1 table $ROUTE_TABLE"
+        exec_cmd "ip route replace default via $gateway dev net1 table $ROUTE_TABLE"
+        ip link set dev net1 arp on
+        exec_cmd "arping -c 3 -s $eip_without_prefix $gateway"
+    done
+}
+
+function add_dnat() {
+    for rule in $@
+    do
+        arr=(${rule//,/ })
+        eip=(${arr[0]//\// })
+        dport=${arr[1]}
+        protocol=${arr[2]}
+        internalIp=${arr[3]}
+        internalPort=${arr[4]}
+        defaultGateway=${arr[5]}
+        # check if already exist
+        iptables-save  | grep "PREROUTING" | grep "\-d $eip" | grep "p $protocol" | grep "dport $dport"| grep "destination $internalIp:$internalPort" && exit 0
+        exec_cmd "iptables -t nat -A PREROUTING -p $protocol -d $eip --dport $dport -j DNAT --to-destination $internalIp:$internalPort"
+
+        exec_cmd "ip route replace $internalIp via $defaultGateway table $ROUTE_TABLE"
+
+        iptables-save  | grep "POSTROUTING" | grep "\-d $internalIp" | grep "MASQUERADE" && exit 0
+        exec_cmd "iptables -t nat -I POSTROUTING -d $internalIp -j MASQUERADE"
+    done
+}
+
+rules=${@:2:${#}}
+opt=$1
+case $opt in
+ init)
+        echo "init $rules"
+        init $rules
+        ;;
+ eip-add)
+        echo "eip-add $rules"
+        add_eip $rules
+        ;;
+ dnat-add)
+        echo "dnat-add $rules"
+        add_dnat $rules
+        ;;
+ *)
+        echo "Usage: $0 [init|eip-add|dnat-add] ..."
+        exit 1
+        ;;
+esac

pkg/controller/config.go

@@ -78,6 +78,7 @@ type Configuration struct {
 	EnableExternalVpc bool
 	EnableEcmp        bool
 	EnableKeepVmIP    bool
+	EnableLbSvc       bool
 
 	ExternalGatewayConfigNS string
 	ExternalGatewayNet      string
@@ -134,6 +135,7 @@ func ParseFlags() (*Configuration, error) {
 		argEnableExternalVpc    = pflag.Bool("enable-external-vpc", true, "Enable external vpc support")
 		argEnableEcmp           = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet")
 		argKeepVmIP             = pflag.Bool("keep-vm-ip", false, "Whether to keep ip for kubevirt pod when pod is rebuild")
+		argEnableLbSvc          = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service")
 
 		argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system")
 		argExternalGatewayNet      = pflag.String("external-gateway-net", "external", "The name of the external network which mappings with an ovs bridge, default: external")
@@ -206,6 +208,7 @@ func ParseFlags() (*Configuration, error) {
 		NodePgProbeTime:               *argNodePgProbeTime,
 		GCInterval:                    *argGCInterval,
 		InspectInterval:               *argInspectInterval,
+		EnableLbSvc:                   *argEnableLbSvc,
 	}
 
 	if config.NetworkType == util.NetworkTypeVlan && config.DefaultHostInterface == "" {

pkg/controller/controller.go

@@ -135,6 +135,7 @@ type Controller struct {
 
 	servicesLister     v1.ServiceLister
 	serviceSynced      cache.InformerSynced
+	addServiceQueue    workqueue.RateLimitingInterface
 	deleteServiceQueue workqueue.RateLimitingInterface
 	updateServiceQueue workqueue.RateLimitingInterface
 
@@ -307,6 +308,7 @@ func NewController(config *Configuration) *Controller {
 
 		servicesLister:     serviceInformer.Lister(),
 		serviceSynced:      serviceInformer.Informer().HasSynced,
+		addServiceQueue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AddService"),
 		deleteServiceQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DeleteService"),
 		updateServiceQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "UpdateService"),
 
@@ -559,6 +561,7 @@ func (c *Controller) shutdown() {
 	c.updateNodeQueue.ShutDown()
 	c.deleteNodeQueue.ShutDown()
 
+	c.addServiceQueue.ShutDown()
 	c.deleteServiceQueue.ShutDown()
 	c.updateServiceQueue.ShutDown()
 	c.updateEndpointQueue.ShutDown()
@@ -677,6 +680,7 @@ func (c *Controller) startWorkers(stopCh <-chan struct{}) {
 	go wait.Until(c.runUpdateProviderNetworkWorker, time.Second, stopCh)
 
 	if c.config.EnableLb {
+		go wait.Until(c.runAddServiceWorker, time.Second, stopCh)
 		// run in a single worker to avoid delete the last vip, which will lead ovn to delete the loadbalancer
 		go wait.Until(c.runDeleteServiceWorker, time.Second, stopCh)
 	}

pkg/controller/gc.go

@@ -34,6 +34,7 @@ func (c *Controller) gc() error {
 		c.gcVpcNatGateway,
 		c.gcLogicalRouterPort,
 		c.gcVip,
+		c.gcLbSvcPods,
 	}
 	for _, gcFunc := range gcFunctions {
 		if err := gcFunc(); err != nil {
@@ -722,3 +723,43 @@ func (c *Controller) getVmLsps() []string {
 
 	return vmLsps
 }
+
+func (c *Controller) gcLbSvcPods() error {
+	klog.Infof("start to gc lb svc pods")
+	nss, err := c.namespacesLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list namespaces, %v", err)
+		return err
+	}
+
+	for _, ns := range nss {
+		dps, err := c.config.KubeClient.AppsV1().Deployments(ns.Name).List(context.Background(), metav1.ListOptions{})
+		if err != nil {
+			if !k8serrors.IsNotFound(err) {
+				klog.Errorf("failed to list lb svc deployment in namespace %s, %v", ns.Name, err)
+			}
+			continue
+		}
+
+		for _, dp := range dps.Items {
+			if !strings.HasPrefix(dp.Name, "lb-svc-") {
+				continue
+			}
+			if _, ok := dp.Spec.Template.Labels["service"]; !ok {
+				continue
+			}
+
+			svcName := strings.TrimPrefix(dp.Name, "lb-svc-")
+			_, err := c.servicesLister.Services(ns.Name).Get(svcName)
+			if err != nil && k8serrors.IsNotFound(err) {
+				klog.Infof("gc lb svc deployment %s in ns %s", dp.Name, ns.Name)
+				if err := c.config.KubeClient.AppsV1().Deployments(ns.Name).Delete(context.Background(), dp.Name, metav1.DeleteOptions{}); err != nil {
+					if !k8serrors.IsNotFound(err) {
+						klog.Errorf("failed to delete lb svc deployment in namespace %s, %v", ns.Name, err)
+					}
+				}
+			}
+		}
+	}
+	return nil
+}