refactor kubevirt vm e2e (#3914)

Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
kubeovn · Apr 15, 2024 · 36025b5 · 36025b5
1 parent b32f607
commit 36025b5
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 121 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -1095,7 +1095,10 @@ jobs:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
           E2E_IP_FAMILY: ${{ matrix.ip-family }}
           E2E_NETWORK_MODE: ${{ matrix.mode }}
-        run: make kube-ovn-conformance-e2e
+        run: |
+          make kube-ovn-conformance-e2e
+          make kind-install-kubevirt
+          make kube-ovn-kubevirt-e2e
 
       - name: kubectl ko log
         if: failure()
@@ -1671,101 +1674,6 @@ jobs:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
         run: make kube-ovn-lb-svc-conformance-e2e
 
-  kubevirt-e2e:
-    name: Kubevirt VM E2E
-    needs:
-      - build-kube-ovn
-      - build-e2e-binaries
-    runs-on: ubuntu-22.04
-    timeout-minutes: 15
-    steps:
-      - uses: jlumbroso/free-disk-space@v1.3.1
-        with:
-          android: true
-          dotnet: true
-          haskell: true
-          docker-images: false
-          large-packages: false
-          tool-cache: false
-          swap-storage: false
-
-      - uses: actions/checkout@v4
-
-      - name: Create the default branch directory
-        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
-        run: mkdir -p test/e2e/source
-
-      - name: Check out the default branch
-        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.repository.default_branch }}
-          fetch-depth: 1
-          path: test/e2e/source
-
-      - name: Export E2E directory
-        run: |
-          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
-            echo "E2E_DIR=." >> "$GITHUB_ENV"
-          else
-            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
-          fi
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION || '' }}
-          go-version-file: ${{ env.E2E_DIR }}/go.mod
-          check-latest: true
-          cache: false
-
-      - name: Export Go full version
-        run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"
-
-      - name: Go cache
-        uses: actions/cache/restore@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
-          restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
-
-      - name: Install kind
-        uses: helm/kind-action@v1.9.0
-        with:
-          version: ${{ env.KIND_VERSION }}
-          install_only: true
-
-      - name: Install ginkgo
-        working-directory: ${{ env.E2E_DIR }}
-        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
-
-      - name: Download kube-ovn image
-        uses: actions/download-artifact@v4
-        with:
-          name: kube-ovn
-
-      - name: Load images
-        run: |
-          docker load -i kube-ovn.tar
-
-      - name: Create kind cluster
-        run: |
-          sudo pip3 install j2cli
-          sudo pip3 install "j2cli[yaml]"
-          sudo PATH=~/.local/bin:$PATH make kind-init
-          sudo cp -r /root/.kube/ ~/.kube/
-          sudo chown -R $(id -un). ~/.kube/
-
-      - name: Install Kube-OVN
-        run: make kind-install-kubevirt
-
-      - name: Run E2E
-        working-directory: ${{ env.E2E_DIR }}
-        env:
-          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
-        run: make kube-ovn-kubevirt-e2e
-
   webhook-e2e:
     name: Webhook E2E
     needs:
@@ -2507,7 +2415,6 @@ jobs:
       - no-np-test
       - cilium-chaining-e2e
       - kube-ovn-ha-e2e
-      - kubevirt-e2e
       - kube-ovn-submariner-conformance-e2e
     if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-22.04

diff --git a/Makefile b/Makefile
@@ -42,10 +42,8 @@ KUBEVIRT_API_IMAGE = quay.io/kubevirt/virt-api:$(KUBEVIRT_VERSION)
 KUBEVIRT_CONTROLLER_IMAGE = quay.io/kubevirt/virt-controller:$(KUBEVIRT_VERSION)
 KUBEVIRT_HANDLER_IMAGE = quay.io/kubevirt/virt-handler:$(KUBEVIRT_VERSION)
 KUBEVIRT_LAUNCHER_IMAGE = quay.io/kubevirt/virt-launcher:$(KUBEVIRT_VERSION)
-KUBEVIRT_TEST_IMAGE = quay.io/kubevirt/cirros-container-disk-demo
 KUBEVIRT_OPERATOR_YAML = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION)/kubevirt-operator.yaml
 KUBEVIRT_CR_YAML = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION)/kubevirt-cr.yaml
-KUBEVIRT_TEST_YAML = https://kubevirt.io/labs/manifests/vm.yaml
 
 CILIUM_VERSION = 1.14.6
 CILIUM_IMAGE_REPO = quay.io/cilium/cilium
@@ -760,24 +758,22 @@ kind-install-vpc-nat-gw:
 	@$(MAKE) kind-install-multus
 
 .PHONY: kind-install-kubevirt
-kind-install-kubevirt: kind-install
+kind-install-kubevirt:
 	$(call kind_load_image,kube-ovn,$(KUBEVIRT_OPERATOR_IMAGE),1)
 	$(call kind_load_image,kube-ovn,$(KUBEVIRT_API_IMAGE),1)
 	$(call kind_load_image,kube-ovn,$(KUBEVIRT_CONTROLLER_IMAGE),1)
 	$(call kind_load_image,kube-ovn,$(KUBEVIRT_HANDLER_IMAGE),1)
 	$(call kind_load_image,kube-ovn,$(KUBEVIRT_LAUNCHER_IMAGE),1)
-	$(call kind_load_image,kube-ovn,$(KUBEVIRT_TEST_IMAGE),1)
 
 	kubectl apply -f "$(KUBEVIRT_OPERATOR_YAML)"
 	kubectl apply -f "$(KUBEVIRT_CR_YAML)"
-	kubectl rollout status deployment/virt-operator -n kubevirt --timeout 120s
-	echo "wait kubevirt releated pod running ..."
-	sleep 60
-
-	kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
-	kubectl apply -f "$(KUBEVIRT_TEST_YAML)"
-	sleep 5
-	kubectl patch vm testvm --type=merge --patch '{"spec":{"running":true}}'
+	kubectl -n kubevirt scale deploy virt-operator --replicas=1
+	kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch \
+		'{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}},"infra":{"replicas":1}}}'
+	$(call kubectl_wait_exist_and_ready,kubevirt,deployment,virt-operator)
+	$(call kubectl_wait_exist_and_ready,kubevirt,deployment,virt-api)
+	$(call kubectl_wait_exist_and_ready,kubevirt,deployment,virt-controller)
+	$(call kubectl_wait_exist_and_ready,kubevirt,daemonset,virt-handler)
 
 .PHONY: kind-install-lb-svc
 kind-install-lb-svc:

diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -640,7 +640,7 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
 		}
 		pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true"
 		if isVMPod && c.config.EnableKeepVMIP {
-			pod.Annotations[fmt.Sprintf(util.VMTemplate, podNet.ProviderName)] = vmName
+			pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, podNet.ProviderName)] = vmName
 			if err := c.changeVMSubnet(vmName, namespace, podNet.ProviderName, subnet.Name); err != nil {
 				klog.Errorf("change subnet of pod %s/%s to %s failed: %v", namespace, name, subnet.Name, err)
 				return nil, err

diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go
@@ -524,8 +524,8 @@ func (c *Controller) handlePod(key string) error {
 	}
 
 	podName := pod.Name
-	if pod.Annotations[fmt.Sprintf(util.VMTemplate, util.OvnProvider)] != "" {
-		podName = pod.Annotations[fmt.Sprintf(util.VMTemplate, util.OvnProvider)]
+	if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)] != "" {
+		podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)]
 	}
 
 	// set default nic bandwidth
@@ -551,8 +551,8 @@ func (c *Controller) handlePod(key string) error {
 	}
 	for _, multiNet := range attachNets {
 		provider := fmt.Sprintf("%s.%s.ovn", multiNet.Name, multiNet.Namespace)
-		if pod.Annotations[fmt.Sprintf(util.VMTemplate, provider)] != "" {
-			podName = pod.Annotations[fmt.Sprintf(util.VMTemplate, provider)]
+		if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, provider)] != "" {
+			podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, provider)]
 		}
 		if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)] == "true" {
 			ifaceID = ovs.PodNameToPortName(podName, pod.Namespace, provider)

diff --git a/pkg/daemon/controller_windows.go b/pkg/daemon/controller_windows.go
@@ -158,8 +158,8 @@ func (c *Controller) handlePod(key string) error {
 	}
 
 	podName := pod.Name
-	if pod.Annotations[fmt.Sprintf(util.VMTemplate, util.OvnProvider)] != "" {
-		podName = pod.Annotations[fmt.Sprintf(util.VMTemplate, util.OvnProvider)]
+	if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)] != "" {
+		podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)]
 	}
 
 	// set default nic bandwidth
@@ -180,8 +180,8 @@ func (c *Controller) handlePod(key string) error {
 	}
 	for _, multiNet := range attachNets {
 		provider := fmt.Sprintf("%s.%s.ovn", multiNet.Name, multiNet.Namespace)
-		if pod.Annotations[fmt.Sprintf(util.VMTemplate, provider)] != "" {
-			podName = pod.Annotations[fmt.Sprintf(util.VMTemplate, provider)]
+		if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, provider)] != "" {
+			podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, provider)]
 		}
 		if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)] == "true" {
 			ifaceID = ovs.PodNameToPortName(podName, pod.Namespace, provider)

diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go
@@ -129,7 +129,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 		loss = pod.Annotations[fmt.Sprintf(util.NetemQosLossAnnotationTemplate, podRequest.Provider)]
 		jitter = pod.Annotations[fmt.Sprintf(util.NetemQosJitterAnnotationTemplate, podRequest.Provider)]
 		providerNetwork = pod.Annotations[fmt.Sprintf(util.ProviderNetworkTemplate, podRequest.Provider)]
-		vmName = pod.Annotations[fmt.Sprintf(util.VMTemplate, podRequest.Provider)]
+		vmName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, podRequest.Provider)]
 		ipAddr = util.GetIPAddrWithMask(ip, cidr)
 		if s := pod.Annotations[fmt.Sprintf(util.RoutesAnnotationTemplate, podRequest.Provider)]; s != "" {
 			if err = json.Unmarshal([]byte(s), &routes); err != nil {
@@ -461,7 +461,7 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon
 		default:
 			nicType = pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podRequest.Provider)]
 		}
-		vmName := pod.Annotations[fmt.Sprintf(util.VMTemplate, podRequest.Provider)]
+		vmName := pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, podRequest.Provider)]
 		if vmName != "" {
 			podRequest.PodName = vmName
 		}

diff --git a/pkg/util/const.go b/pkg/util/const.go
@@ -19,6 +19,7 @@ const (
 	FipFinalizer         = "ovn.kubernetes.io/fip"
 	VipAnnotation        = "ovn.kubernetes.io/vip"
 	ChassisAnnotation    = "ovn.kubernetes.io/chassis"
+	VMAnnotation         = "ovn.kubernetes.io/virtualmachine"
 
 	ExternalIPAnnotation         = "ovn.kubernetes.io/external_ip"
 	ExternalMacAnnotation        = "ovn.kubernetes.io/external_mac"
@@ -81,7 +82,7 @@ const (
 	ProviderNetworkMtuTemplate        = "%s.provider-network.kubernetes.io/mtu"
 	MirrorControlAnnotationTemplate   = "%s.kubernetes.io/mirror"
 	PodNicAnnotationTemplate          = "%s.kubernetes.io/pod_nic_type"
-	VMTemplate                        = "%s.kubernetes.io/virtualmachine"
+	VMAnnotationTemplate              = "%s.kubernetes.io/virtualmachine"
 
 	ExcludeIpsAnnotation = "ovn.kubernetes.io/exclude_ips"