add install for webhook (#2460)

kubeovn · Mar 10, 2023 · 4d850e0 · 4d850e0
1 parent f17b434
commit 4d850e0
Show file tree

Hide file tree

Showing 8 changed files with 466 additions and 8 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -1106,6 +1106,86 @@ jobs:
         working-directory: ${{ env.E2E_DIR }}
         run: make kube-ovn-kubevirt-e2e
 
+  webhook-e2e:
+    name: Webhook E2E
+    needs:
+      - build-kube-ovn
+      - build-e2e-binaries
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create the default branch directory
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        run: mkdir -p test/e2e/source
+
+      - name: Check out the default branch
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 1
+          path: test/e2e/source
+
+      - name: Export E2E directory
+        run: |
+          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
+            echo "E2E_DIR=." >> "$GITHUB_ENV"
+          else
+            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
+          fi
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: ${{ env.E2E_DIR }}/go.mod
+          check-latest: true
+
+      - name: Export Go full version
+        run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"
+
+      - name: Go cache
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
+          restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
+
+      - name: Install kind
+        uses: helm/kind-action@v1
+        with:
+          install_only: true
+
+      - name: Install ginkgo
+        working-directory: ${{ env.E2E_DIR }}
+        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+
+      - name: Download kube-ovn image
+        uses: actions/download-artifact@v3
+        with:
+          name: kube-ovn
+
+      - name: Load images
+        run: |
+          docker load -i kube-ovn.tar
+
+      - name: Create kind cluster
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chown -R $(id -un). ~/.kube/
+
+      - name: Install Kube-OVN
+        run: make kind-install-webhook
+
+      - name: Run E2E
+        working-directory: ${{ env.E2E_DIR }}
+        run: make kube-ovn-webhook-e2e
+
   installation-compatibility-test:
     name: Installation Compatibility Test
     needs: build-kube-ovn

diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml
@@ -814,6 +814,73 @@ jobs:
       - name: Run E2E
         run: make kube-ovn-kubevirt-e2e
 
+  webhook-e2e:
+    name: Webhook E2E
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - master
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create branch directory
+        run: mkdir -p test/e2e/kube-ovn/branches/${{ matrix.branch }}
+
+      - name: Check out branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ matrix.branch }}
+          fetch-depth: 1
+          path: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          check-latest: true
+
+      - name: Export Go full version
+        run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"
+
+      - name: Go cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
+
+      - name: Install kind
+        uses: helm/kind-action@v1
+        with:
+          install_only: true
+
+      - name: Install ginkgo
+        working-directory: ${{ env.E2E_DIR }}
+        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+
+      - name: Create kind cluster
+        working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chown -R $(id -un). ~/.kube/
+
+      - name: Install Kube-OVN
+        working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+        run: |
+          version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"')
+          docker pull kubeovn/kube-ovn:$version
+          VERSION=$version make kind-install-webhook
+
+      - name: Run E2E
+        run: make kube-ovn-webhook-e2e
+
   installation-compatibility-test:
     name: Installation Compatibility Test
     runs-on: ubuntu-22.04

diff --git a/Makefile b/Makefile
@@ -17,19 +17,26 @@ CHART_UPGRADE_RESTART_OVS=$(shell echo $${CHART_UPGRADE_RESTART_OVS:-false})
 MULTUS_IMAGE = ghcr.io/k8snetworkplumbingwg/multus-cni:stable
 MULTUS_YAML = https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
 
-KUBEVIRT_OPERATOR_IMAGE = quay.io/kubevirt/virt-operator:v0.58.0
-KUBEVIRT_API_IMAGE = quay.io/kubevirt/virt-api:v0.58.0
-KUBEVIRT_CONTROLLER_IMAGE = quay.io/kubevirt/virt-controller:v0.58.0
-KUBEVIRT_HANDLER_IMAGE = quay.io/kubevirt/virt-handler:v0.58.0
-KUBEVIRT_LAUNCHER_IMAGE = quay.io/kubevirt/virt-launcher:v0.58.0
+KUBEVIRT_VERSION = v0.58.0
+KUBEVIRT_OPERATOR_IMAGE = quay.io/kubevirt/virt-operator:$(KUBEVIRT_VERSION)
+KUBEVIRT_API_IMAGE = quay.io/kubevirt/virt-api:$(KUBEVIRT_VERSION)
+KUBEVIRT_CONTROLLER_IMAGE = quay.io/kubevirt/virt-controller:$(KUBEVIRT_VERSION)
+KUBEVIRT_HANDLER_IMAGE = quay.io/kubevirt/virt-handler:$(KUBEVIRT_VERSION)
+KUBEVIRT_LAUNCHER_IMAGE = quay.io/kubevirt/virt-launcher:$(KUBEVIRT_VERSION)
 KUBEVIRT_TEST_IMAGE = quay.io/kubevirt/cirros-container-disk-demo
-KUBEVIRT_OPERATOR_YAML = https://github.com/kubevirt/kubevirt/releases/download/v0.58.0/kubevirt-operator.yaml
-KUBEVIRT_CR_YAML = https://github.com/kubevirt/kubevirt/releases/download/v0.58.0/kubevirt-cr.yaml
+KUBEVIRT_OPERATOR_YAML = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION)/kubevirt-operator.yaml
+KUBEVIRT_CR_YAML = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION)/kubevirt-cr.yaml
 KUBEVIRT_TEST_YAML = https://kubevirt.io/labs/manifests/vm.yaml
 
 CILIUM_VERSION = 1.12.7
 CILIUM_IMAGE_REPO = quay.io/cilium/cilium
 
+CERT_MANAGER_VERSION = v1.11.0
+CERT_MANAGER_CONTROLLER = quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION)
+CERT_MANAGER_CAINJECTOR = quay.io/jetstack/cert-manager-cainjector:$(CERT_MANAGER_VERSION)
+CERT_MANAGER_WEBHOOK = quay.io/jetstack/cert-manager-webhook:$(CERT_MANAGER_VERSION)
+CERT_MANAGER_YAML = https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
+
 VPC_NAT_GW_IMG = $(REGISTRY)/vpc-nat-gateway:$(VERSION)
 
 E2E_NETWORK = bridge
@@ -565,7 +572,7 @@ kind-install-kubevirt: kind-load-image kind-untaint-control-plane
 
 	kubectl apply -f "$(KUBEVIRT_OPERATOR_YAML)"
 	kubectl apply -f "$(KUBEVIRT_CR_YAML)"
-	kubectl rollout status deployment/virt-operator -n kubevirt
+	kubectl rollout status deployment/virt-operator -n kubevirt --timeout 120s
 	echo "wait kubevirt releated pod running ..."
 	sleep 60
 
@@ -582,6 +589,26 @@ kind-install-lb-svc: kind-load-image kind-untaint-control-plane
 	ENABLE_LB_SVC=true CNI_CONFIG_PRIORITY=10 bash
 	kubectl describe no
 
+.PHONY: kind-install-webhook
+kind-install-webhook: kind-load-image kind-untaint-control-plane
+	$(call docker_ensure_image_exists,$(CERT_MANAGER_CONTROLLER))
+	$(call kind_load_image,kube-ovn,$(CERT_MANAGER_CONTROLLER))
+	$(call docker_ensure_image_exists,$(CERT_MANAGER_CAINJECTOR))
+	$(call kind_load_image,kube-ovn,$(CERT_MANAGER_CAINJECTOR))
+	$(call docker_ensure_image_exists,$(CERT_MANAGER_WEBHOOK))
+	$(call kind_load_image,kube-ovn,$(CERT_MANAGER_WEBHOOK))
+
+	sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install.sh | bash
+	kubectl describe no
+
+	kubectl apply -f "$(CERT_MANAGER_YAML)"
+	kubectl rollout status deployment/cert-manager -n cert-manager --timeout 120s
+	kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout 120s
+	kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout 120s
+
+	kubectl apply -f yamls/webhook.yaml
+	kubectl rollout status deployment/kube-ovn-webhook -n kube-system --timeout 120s
+
 .PHONY: kind-install-cilium-chaining
 kind-install-cilium-chaining: kind-load-image kind-untaint-control-plane
 	$(eval KUBERNETES_SERVICE_HOST = $(shell kubectl get nodes kube-ovn-control-plane -o jsonpath='{.status.addresses[0].address}'))

diff --git a/Makefile.e2e b/Makefile.e2e
@@ -170,3 +170,12 @@ kube-ovn-kubevirt-e2e:
 	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
 	ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \
 		--focus=CNI:Kube-OVN ./test/e2e/kubevirt/kubevirt.test
+
+.PHONY: kube-ovn-webhook-e2e
+kube-ovn-webhook-e2e:
+	ginkgo build ./test/e2e/webhook
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
+	ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \
+		--focus=CNI:Kube-OVN ./test/e2e/webhook/webhook.test
diff --git a/pkg/util/validator.go b/pkg/util/validator.go
@@ -150,6 +150,13 @@ func ValidatePodNetwork(annotations map[string]string) error {
 	ipPool := annotations[IpPoolAnnotation]
 	if ipPool != "" {
 		for _, ips := range strings.Split(ipPool, ";") {
+			if cidrStr := annotations[CidrAnnotation]; cidrStr != "" {
+				if !CIDRContainIP(cidrStr, ips) {
+					errors = append(errors, fmt.Errorf("%s not in cidr %s", ips, cidrStr))
+					continue
+				}
+			}
+
 			for _, ip := range strings.Split(ips, ",") {
 				if net.ParseIP(strings.TrimSpace(ip)) == nil {
 					errors = append(errors, fmt.Errorf("%s in %s is not a valid address", ip, IpPoolAnnotation))

diff --git a/test/e2e/webhook/e2e_test.go b/test/e2e/webhook/e2e_test.go
@@ -0,0 +1,37 @@
+package webhook
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/test/e2e"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/config"
+
+	"github.com/onsi/ginkgo/v2"
+
+	// Import tests.
+	_ "github.com/kubeovn/kube-ovn/test/e2e/webhook/pod"
+	_ "github.com/kubeovn/kube-ovn/test/e2e/webhook/subnet"
+)
+
+func init() {
+	klog.SetOutput(ginkgo.GinkgoWriter)
+
+	// Register flags.
+	config.CopyFlags(config.Flags, flag.CommandLine)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	framework.RegisterClusterFlags(flag.CommandLine)
+}
+
+func TestE2E(t *testing.T) {
+	if framework.TestContext.KubeConfig == "" {
+		framework.TestContext.KubeConfig = filepath.Join(os.Getenv("HOME"), ".kube", "config")
+	}
+	framework.AfterReadingAllFlags(&framework.TestContext)
+
+	e2e.RunE2ETests(t)
+}