diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 0181119f45c..ee1b298cea6 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -1026,6 +1026,86 @@ jobs: working-directory: ${{ env.E2E_DIR }} run: make kube-ovn-lb-svc-conformance-e2e + kubevirt-e2e: + name: Kubevirt vm E2E + needs: + - build-kube-ovn + - build-e2e-binaries + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v3 + + - name: Create the default branch directory + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + run: mkdir -p test/e2e/source + + - name: Check out the default branch + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + uses: actions/checkout@v3 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + path: test/e2e/source + + - name: Export E2E directory + run: | + if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then + echo "E2E_DIR=." >> "$GITHUB_ENV" + else + echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" + fi + + - uses: actions/setup-go@v3 + with: + go-version-file: ${{ env.E2E_DIR }}/go.mod + check-latest: true + + - name: Export Go full version + run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" + + - name: Go cache + uses: actions/cache/restore@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} + restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- + + - name: Install kind + uses: helm/kind-action@v1 + with: + install_only: true + + - name: Install ginkgo + working-directory: ${{ env.E2E_DIR }} + run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo + + - name: Download kube-ovn image + uses: actions/download-artifact@v3 + with: + name: kube-ovn + + - name: Load images + run: | + docker load -i kube-ovn.tar + + - name: Create kind cluster + run: | + sudo pip3 install j2cli + sudo pip3 install "j2cli[yaml]" + sudo PATH=~/.local/bin:$PATH make kind-init + sudo cp -r /root/.kube/ ~/.kube/ + sudo chown -R $(id -un). ~/.kube/ + + - name: Install Kube-OVN + run: make kind-install-kubevirt + + - name: Run E2E + working-directory: ${{ env.E2E_DIR }} + run: make kube-ovn-kubevirt-e2e + installation-compatibility-test: name: Installation Compatibility Test needs: build-kube-ovn @@ -1271,6 +1351,7 @@ jobs: - no-np-test - cilium-chaining-e2e - kube-ovn-security-e2e + - kubevirt-e2e if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml index fb27cc6265f..dd3c6eec108 100644 --- a/.github/workflows/scheduled-e2e.yaml +++ b/.github/workflows/scheduled-e2e.yaml @@ -747,6 +747,73 @@ jobs: - name: Run E2E run: make kube-ovn-lb-svc-conformance-e2e + kubevirt-e2e: + name: Kubevirt vm E2E + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + branch: + - master + steps: + - uses: actions/checkout@v3 + + - name: Create branch directory + run: mkdir -p test/e2e/kube-ovn/branches/${{ matrix.branch }} + + - name: Check out branch + uses: actions/checkout@v3 + with: + ref: ${{ matrix.branch }} + fetch-depth: 1 + path: test/e2e/kube-ovn/branches/${{ matrix.branch }} + + - uses: actions/setup-go@v3 + with: + go-version-file: go.mod + check-latest: true + + - name: Export Go full version + run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" + + - name: Go cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- + + - name: Install kind + uses: helm/kind-action@v1 + with: + install_only: true + + - name: Install ginkgo + working-directory: ${{ env.E2E_DIR }} + run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo + + - name: Create kind cluster + working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }} + run: | + sudo pip3 install j2cli + sudo pip3 install "j2cli[yaml]" + sudo PATH=~/.local/bin:$PATH make kind-init + sudo cp -r /root/.kube/ ~/.kube/ + sudo chown -R $(id -un). ~/.kube/ + + - name: Install Kube-OVN + working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }} + run: | + version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"') + docker pull kubeovn/kube-ovn:$version + VERSION=$version make kind-install-kubevirt + + - name: Run E2E + run: make kube-ovn-kubevirt-e2e + installation-compatibility-test: name: Installation Compatibility Test runs-on: ubuntu-22.04 diff --git a/Makefile b/Makefile index e79a5f96d4d..f56b74770ad 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,16 @@ CONTROL_PLANE_TAINTS = node-role.kubernetes.io/master node-role.kubernetes.io/co MULTUS_IMAGE = ghcr.io/k8snetworkplumbingwg/multus-cni:stable MULTUS_YAML = https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml +KUBEVIRT_OPERATOR_IMAGE = quay.io/kubevirt/virt-operator:v0.58.0 +KUBEVIRT_API_IMAGE = quay.io/kubevirt/virt-api:v0.58.0 +KUBEVIRT_CONTROLLER_IMAGE = quay.io/kubevirt/virt-controller:v0.58.0 +KUBEVIRT_HANDLER_IMAGE = quay.io/kubevirt/virt-handler:v0.58.0 +KUBEVIRT_LAUNCHER_IMAGE = quay.io/kubevirt/virt-launcher:v0.58.0 +KUBEVIRT_TEST_IMAGE = quay.io/kubevirt/cirros-container-disk-demo +KUBEVIRT_OPERATOR_YAML = https://github.com/kubevirt/kubevirt/releases/download/v0.58.0/kubevirt-operator.yaml +KUBEVIRT_CR_YAML = https://github.com/kubevirt/kubevirt/releases/download/v0.58.0/kubevirt-cr.yaml +KUBEVIRT_TEST_YAML = https://kubevirt.io/labs/manifests/vm.yaml + CILIUM_VERSION = 1.12.7 CILIUM_IMAGE_REPO = quay.io/cilium/cilium @@ -531,6 +541,35 @@ kind-install-multus: kubectl apply -f "$(MULTUS_YAML)" kubectl -n kube-system rollout status ds kube-multus-ds +.PHONY: kind-install-kubevirt +kind-install-kubevirt: kind-load-image kind-untaint-control-plane + $(call docker_ensure_image_exists,$(KUBEVIRT_OPERATOR_IMAGE)) + $(call kind_load_image,kube-ovn,$(KUBEVIRT_OPERATOR_IMAGE)) + $(call docker_ensure_image_exists,$(KUBEVIRT_API_IMAGE)) + $(call kind_load_image,kube-ovn,$(KUBEVIRT_API_IMAGE)) + $(call docker_ensure_image_exists,$(KUBEVIRT_CONTROLLER_IMAGE)) + $(call kind_load_image,kube-ovn,$(KUBEVIRT_CONTROLLER_IMAGE)) + $(call docker_ensure_image_exists,$(KUBEVIRT_HANDLER_IMAGE)) + $(call kind_load_image,kube-ovn,$(KUBEVIRT_HANDLER_IMAGE)) + $(call docker_ensure_image_exists,$(KUBEVIRT_LAUNCHER_IMAGE)) + $(call kind_load_image,kube-ovn,$(KUBEVIRT_LAUNCHER_IMAGE)) + $(call docker_ensure_image_exists,$(KUBEVIRT_TEST_IMAGE)) + $(call kind_load_image,kube-ovn,$(KUBEVIRT_TEST_IMAGE)) + + sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install.sh | bash + kubectl describe no + + kubectl apply -f "$(KUBEVIRT_OPERATOR_YAML)" + kubectl apply -f "$(KUBEVIRT_CR_YAML)" + kubectl rollout status deployment/virt-operator -n kubevirt + echo "wait kubevirt releated pod running ..." + sleep 60 + + kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}' + kubectl apply -f "$(KUBEVIRT_TEST_YAML)" + sleep 5 + kubectl patch vm testvm --type=merge --patch '{"spec":{"running":true}}' + .PHONY: kind-install-lb-svc kind-install-lb-svc: kind-load-image kind-untaint-control-plane $(call kind_load_image,kube-ovn,$(VPC_NAT_GW_IMG)) diff --git a/Makefile.e2e b/Makefile.e2e index 19c9ce1acdb..ca58a77025e 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -59,6 +59,7 @@ e2e-build: ginkgo build ./test/e2e/lb-svc ginkgo build ./test/e2e/ovn-eip ginkgo build ./test/e2e/security + ginkgo build ./test/e2e/kubevirt .PHONY: k8s-conformance-e2e k8s-conformance-e2e: @@ -160,3 +161,12 @@ kube-ovn-security-e2e: E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \ --focus=CNI:Kube-OVN ./test/e2e/security/security.test + +.PHONY: kube-ovn-kubevirt-e2e +kube-ovn-kubevirt-e2e: + ginkgo build ./test/e2e/kubevirt + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \ + --focus=CNI:Kube-OVN ./test/e2e/kubevirt/kubevirt.test diff --git a/test/e2e/kubevirt/e2e_test.go b/test/e2e/kubevirt/e2e_test.go new file mode 100644 index 00000000000..5038425445e --- /dev/null +++ b/test/e2e/kubevirt/e2e_test.go @@ -0,0 +1,90 @@ +package kubevirt + +import ( + "context" + "flag" + "os" + "path/filepath" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + + "github.com/onsi/ginkgo/v2" + + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" +) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + if k8sframework.TestContext.KubeConfig == "" { + k8sframework.TestContext.KubeConfig = filepath.Join(os.Getenv("HOME"), ".kube", "config") + } + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + + e2e.RunE2ETests(t) +} + +var _ = framework.Describe("[group:kubevirt]", func() { + f := framework.NewDefaultFramework("kubevirt") + + var cs clientset.Interface + + ginkgo.BeforeEach(func() { + cs = f.ClientSet + }) + + framework.ConformanceIt("Kubevirt vm pod should keep ip", func() { + f.SkipVersionPriorTo(1, 12, "Only test kubevirt vm keep ip in master") + + ginkgo.By("Get kubevirt vm pod") + podList, err := cs.CoreV1().Pods("default").List(context.TODO(), metav1.ListOptions{ + LabelSelector: "vm.kubevirt.io/name=testvm", + }) + framework.ExpectNoError(err) + framework.ExpectEqual(len(podList.Items), 1) + + ginkgo.By("Validating pod annotations") + pod := podList.Items[0] + framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, "ovn.kubernetes.io/virtualmachine", "testvm") + ipAddr := pod.Annotations[util.IpAddressAnnotation] + + ginkgo.By("Deleting pod " + pod.Name) + err = cs.CoreV1().Pods("default").Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + time.Sleep(10 * time.Second) + + ginkgo.By("Check kubevirt vm pod after rebuild") + podList, err = cs.CoreV1().Pods("default").List(context.TODO(), metav1.ListOptions{ + LabelSelector: "vm.kubevirt.io/name=testvm", + }) + framework.ExpectNoError(err) + framework.ExpectEqual(len(podList.Items), 1) + + ginkgo.By("Validating new pod annotations") + pod = podList.Items[0] + framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, "ovn.kubernetes.io/virtualmachine", "testvm") + + ginkgo.By("Check vm pod ip unchanged" + pod.Name) + ipNewAddr := pod.Annotations[util.IpAddressAnnotation] + framework.ExpectEqual(ipAddr, ipNewAddr) + }) +})