underlay/vlan network refactoring

Changes and new features: 1. Introduce new CRD `ProviderNetwork` for underlay/vlan networking management; 2. Specify provider inetrface on nodes in provider network; 4. Exclude nodes in provider network; 5. Link local IPv6 address and route will NOT be transferred to OVS bridge; 6. Builtin hybrid network support - create/update/delete underlay/vlan networks dynamically; 7. Set MTU of Pod interface to provider interface's MTU on each node; 8. Add new fields in CRD `Vlan`.
kubeovn · Jul 13, 2021 · 0b877b9 · 0b877b9
1 parent 7c529a1
commit 0b877b9
Show file tree

Hide file tree

Showing 56 changed files with 2,975 additions and 521 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -123,9 +123,9 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
-  single-vlan-e2e:
+  single-vlan-e2e-single-nic:
     needs: build
-    name: 1-master-vlan-e2e
+    name: 1-master-vlan-e2e-single-nic
     runs-on: ubuntu-18.04
     timeout-minutes: 30
     steps:
@@ -168,7 +168,7 @@ jobs:
           go get -u github.com/onsi/gomega/...
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
-          make e2e-vlan
+          make e2e-vlan-single-nic
 
   single-node-e2e:
     needs: build
@@ -287,13 +287,65 @@ jobs:
           docker load --input image.tar
           sudo make kind-install-ipv6
 
+  ipv6-vlan-e2e-single-nic:
+    needs: build
+    name: ipv6-vlan-e2e-single-nic
+    runs-on: ubuntu-18.04
+    timeout-minutes: 30
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install Kind
+        env:
+          KIND_VERSION: v0.11.1
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Init Kind
+        run: |
+          pip install j2cli --user
+          pip install "j2cli[yaml]" --user
+          sudo PATH=~/.local/bin:$PATH make kind-init-ipv6
+
+      - name: Download image
+        uses: actions/download-artifact@v2
+        with:
+          name: image
+
+      - name: Load Image
+        run: |
+          docker load --input image.tar
+
+      - name: Install Kube-OVN
+        run: |
+          docker load --input image.tar
+          sudo make kind-install-ipv6-vlan
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+        id: go
+
+      - name: Run E2E
+        run: |
+          go get -u github.com/onsi/ginkgo/ginkgo
+          go get -u github.com/onsi/gomega/...
+          sudo kubectl cluster-info
+          sudo chmod 666 /home/runner/.kube/config
+          make e2e-vlan-single-nic
+
   push:
     needs:
       - single-e2e
-      - single-vlan-e2e
+      - single-vlan-e2e-single-nic
       - single-node-e2e
       - ha-e2e
       - ipv6-e2e
+      - ipv6-vlan-e2e-single-nic
     name: push
     runs-on: ubuntu-18.04
     steps:

diff --git a/Makefile b/Makefile
@@ -145,14 +145,9 @@ kind-install-ipv6:
 
 .PHONY: kind-install-ipv6-vlan
 kind-install-ipv6-vlan:
-	docker network inspect bridge
-	docker network inspect kind
 	$(eval SUBNET = $(shell docker network inspect kind -f "{{(index .IPAM.Config 1).Subnet}}"))
-	$(eval GATEWAY = $(shell docker network inspect kind -f "{{(index .IPAM.Config 1).Gateway}}"))
 	$(eval EXCLUDE_IPS = $(shell docker network inspect kind -f '{{range .Containers}},{{index (split .IPv6Address "/") 0}}{{end}}' | sed 's/^,//'))
-ifeq ($(GATEWAY),)
 	$(eval GATEWAY = $(shell docker exec kube-ovn-worker ip -6 route show default | awk '{print $$3}'))
-endif
 	sed -e 's@^[[:space:]]*POD_CIDR=.*@POD_CIDR="$(SUBNET)"@' \
 		-e 's@^[[:space:]]*POD_GATEWAY=.*@POD_GATEWAY="$(GATEWAY)"@' \
 		-e 's@^[[:space:]]*EXCLUDE_IPS=.*@EXCLUDE_IPS="$(EXCLUDE_IPS)"@' \
@@ -202,13 +197,28 @@ ut:
 
 .PHONY: e2e
 e2e:
+	$(eval NETWORK_BRIDGE = $(shell docker inspect -f '{{json .NetworkSettings.Networks.bridge}}' kube-ovn-control-plane))
+	if [ '$(NETWORK_BRIDGE)' = 'null' ]; then \
+		kind get nodes --name kube-ovn | while read node; do \
+		docker network connect bridge $$node; \
+		done; \
+	fi
+
+	printf "package underlay\n\nvar nodeNetworks = map[string]string{\n" > test/e2e/underlay/network.go
+	kind get nodes --name kube-ovn | while read node; do \
+		printf "\`$$node\`: \`" >> test/e2e/underlay/network.go; \
+		docker inspect -f '{{json .NetworkSettings.Networks.bridge}}' $$node >> test/e2e/underlay/network.go; \
+		printf "\`,\n" >> test/e2e/underlay/network.go; \
+	done
+	echo "}" >> test/e2e/underlay/network.go
+
 	docker pull kubeovn/pause:3.2
 	kind load docker-image --name kube-ovn kubeovn/pause:3.2
 	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e
 
-.PHONY: e2e-vlan
-e2e-vlan:
-	printf "package node\n\nvar networkJSON = []byte(\`" > test/e2e-vlan/node/network.go
-	docker inspect -f '{{json .NetworkSettings.Networks.kind}}' kube-ovn-control-plane >> test/e2e-vlan/node/network.go
-	echo "\`)" >> test/e2e-vlan/node/network.go
-	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e-vlan
+.PHONY: e2e-vlan-single-nic
+e2e-vlan-single-nic:
+	printf "package node\n\nvar networkJSON = []byte(\`" > test/e2e-vlan-single-nic/node/network.go
+	docker inspect -f '{{json .NetworkSettings.Networks.kind}}' kube-ovn-control-plane >> test/e2e-vlan-single-nic/node/network.go
+	echo "\`)" >> test/e2e-vlan-single-nic/node/network.go
+	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e-vlan-single-nic
diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -5,22 +5,22 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"k8s.io/apimachinery/pkg/types"
 	"net/http"
 	_ "net/http/pprof" // #nosec
 	"strings"
 	"time"
 
-	"github.com/kubeovn/kube-ovn/pkg/util"
-	"github.com/kubeovn/kube-ovn/versions"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
-	"github.com/kubeovn/kube-ovn/pkg/daemon"
+	"k8s.io/apimachinery/pkg/types"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog"
 	"k8s.io/sample-controller/pkg/signals"
+
+	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
+	"github.com/kubeovn/kube-ovn/pkg/daemon"
+	"github.com/kubeovn/kube-ovn/pkg/util"
+	"github.com/kubeovn/kube-ovn/versions"
 )
 
 func CmdMain() {
@@ -45,12 +45,6 @@ func CmdMain() {
 		klog.Fatalf("init node gateway failed %v", err)
 	}
 
-	if util.IsNetworkVlan(config.NetworkType) {
-		if err = daemon.InitVlan(config); err != nil {
-			klog.Fatalf("init vlan config failed %v", err)
-		}
-	}
-
 	stopCh := signals.SetupSignalHandler()
 	podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0,
 		kubeinformers.WithTweakListOptions(func(listOption *v1.ListOptions) {

diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh
@@ -5,6 +5,16 @@ for subnet in $(kubectl get subnet -o name); do
   kubectl patch "$subnet" --type='json' -p '[{"op": "replace", "path": "/metadata/finalizers", "value": []}]'
 done
 
+for vlan in $(kubectl get vlan -o name); do
+  kubectl delete $vlan
+done
+
+for pn in $(kubectl get provider-network -o name); do
+  kubectl delete $pn
+done
+
+sleep 3
+
 # Delete Kube-OVN components
 kubectl delete cm ovn-config ovn-ic-config ovn-external-gw-config -n kube-system --ignore-not-found=true
 kubectl delete secret kube-ovn-tls -n kube-system --ignore-not-found=true
@@ -23,7 +33,14 @@ do
   fi
 done
 kubectl delete ds ovs-ovn kube-ovn-pinger -n kube-system --ignore-not-found=true
-kubectl delete crd ips.kubeovn.io subnets.kubeovn.io vlans.kubeovn.io networks.kubeovn.io --ignore-not-found=true
+kubectl delete crd --ignore-not-found=true \
+  ips.kubeovn.io \
+  subnets.kubeovn.io \
+  vpc-nat-gateways.kubeovn.io \
+  vpcs.kubeovn.io \
+  vlans.kubeovn.io \
+  provider-networks.kubeovn.io \
+  networks.kubeovn.io
 
 # Remove annotations/labels in namespaces and nodes
 kubectl annotate no --all ovn.kubernetes.io/cidr-
@@ -58,8 +75,6 @@ for ns in $(kubectl get ns -o name |cut -c 11-); do
   kubectl annotate pod  --all ovn.kubernetes.io/allocated- -n "$ns"
   kubectl annotate pod  --all ovn.kubernetes.io/routed- -n "$ns"
   kubectl annotate pod  --all ovn.kubernetes.io/vlan_id- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/vlan_range- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/network_types- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/provider_interface_name- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/host_interface_name- -n "$ns"
+  kubectl annotate pod  --all ovn.kubernetes.io/network_type- -n "$ns"
+  kubectl annotate pod  --all ovn.kubernetes.io/provider_network- -n "$ns"
 done