Skip to content

Commit

Permalink
add kubevirt install (#2430)
Browse files Browse the repository at this point in the history
  • Loading branch information
hongzhen-ma committed Mar 7, 2023
1 parent e9017e2 commit c84479b
Show file tree
Hide file tree
Showing 5 changed files with 287 additions and 0 deletions.
81 changes: 81 additions & 0 deletions .github/workflows/build-x86-image.yaml
Expand Up @@ -1026,6 +1026,86 @@ jobs:
working-directory: ${{ env.E2E_DIR }}
run: make kube-ovn-lb-svc-conformance-e2e

kubevirt-e2e:
name: Kubevirt vm E2E
needs:
- build-kube-ovn
- build-e2e-binaries
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- uses: actions/checkout@v3

- name: Create the default branch directory
if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
run: mkdir -p test/e2e/source

- name: Check out the default branch
if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
uses: actions/checkout@v3
with:
ref: ${{ github.event.repository.default_branch }}
fetch-depth: 1
path: test/e2e/source

- name: Export E2E directory
run: |
if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
echo "E2E_DIR=." >> "$GITHUB_ENV"
else
echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
fi
- uses: actions/setup-go@v3
with:
go-version-file: ${{ env.E2E_DIR }}/go.mod
check-latest: true

- name: Export Go full version
run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"

- name: Go cache
uses: actions/cache/restore@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-

- name: Install kind
uses: helm/kind-action@v1
with:
install_only: true

- name: Install ginkgo
working-directory: ${{ env.E2E_DIR }}
run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo

- name: Download kube-ovn image
uses: actions/download-artifact@v3
with:
name: kube-ovn

- name: Load images
run: |
docker load -i kube-ovn.tar
- name: Create kind cluster
run: |
sudo pip3 install j2cli
sudo pip3 install "j2cli[yaml]"
sudo PATH=~/.local/bin:$PATH make kind-init
sudo cp -r /root/.kube/ ~/.kube/
sudo chown -R $(id -un). ~/.kube/
- name: Install Kube-OVN
run: make kind-install-kubevirt

- name: Run E2E
working-directory: ${{ env.E2E_DIR }}
run: make kube-ovn-kubevirt-e2e

installation-compatibility-test:
name: Installation Compatibility Test
needs: build-kube-ovn
Expand Down Expand Up @@ -1271,6 +1351,7 @@ jobs:
- no-np-test
- cilium-chaining-e2e
- kube-ovn-security-e2e
- kubevirt-e2e
if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
runs-on: ubuntu-22.04
steps:
Expand Down
67 changes: 67 additions & 0 deletions .github/workflows/scheduled-e2e.yaml
Expand Up @@ -747,6 +747,73 @@ jobs:
- name: Run E2E
run: make kube-ovn-lb-svc-conformance-e2e

kubevirt-e2e:
name: Kubevirt vm E2E
runs-on: ubuntu-22.04
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
branch:
- master
steps:
- uses: actions/checkout@v3

- name: Create branch directory
run: mkdir -p test/e2e/kube-ovn/branches/${{ matrix.branch }}

- name: Check out branch
uses: actions/checkout@v3
with:
ref: ${{ matrix.branch }}
fetch-depth: 1
path: test/e2e/kube-ovn/branches/${{ matrix.branch }}

- uses: actions/setup-go@v3
with:
go-version-file: go.mod
check-latest: true

- name: Export Go full version
run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"

- name: Go cache
uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-

- name: Install kind
uses: helm/kind-action@v1
with:
install_only: true

- name: Install ginkgo
working-directory: ${{ env.E2E_DIR }}
run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo

- name: Create kind cluster
working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
run: |
sudo pip3 install j2cli
sudo pip3 install "j2cli[yaml]"
sudo PATH=~/.local/bin:$PATH make kind-init
sudo cp -r /root/.kube/ ~/.kube/
sudo chown -R $(id -un). ~/.kube/
- name: Install Kube-OVN
working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
run: |
version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"')
docker pull kubeovn/kube-ovn:$version
VERSION=$version make kind-install-kubevirt
- name: Run E2E
run: make kube-ovn-kubevirt-e2e

installation-compatibility-test:
name: Installation Compatibility Test
runs-on: ubuntu-22.04
Expand Down
39 changes: 39 additions & 0 deletions Makefile
Expand Up @@ -15,6 +15,16 @@ CONTROL_PLANE_TAINTS = node-role.kubernetes.io/master node-role.kubernetes.io/co
MULTUS_IMAGE = ghcr.io/k8snetworkplumbingwg/multus-cni:stable
MULTUS_YAML = https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml

KUBEVIRT_OPERATOR_IMAGE = quay.io/kubevirt/virt-operator:v0.58.0
KUBEVIRT_API_IMAGE = quay.io/kubevirt/virt-api:v0.58.0
KUBEVIRT_CONTROLLER_IMAGE = quay.io/kubevirt/virt-controller:v0.58.0
KUBEVIRT_HANDLER_IMAGE = quay.io/kubevirt/virt-handler:v0.58.0
KUBEVIRT_LAUNCHER_IMAGE = quay.io/kubevirt/virt-launcher:v0.58.0
KUBEVIRT_TEST_IMAGE = quay.io/kubevirt/cirros-container-disk-demo
KUBEVIRT_OPERATOR_YAML = https://github.com/kubevirt/kubevirt/releases/download/v0.58.0/kubevirt-operator.yaml
KUBEVIRT_CR_YAML = https://github.com/kubevirt/kubevirt/releases/download/v0.58.0/kubevirt-cr.yaml
KUBEVIRT_TEST_YAML = https://kubevirt.io/labs/manifests/vm.yaml

CILIUM_VERSION = 1.12.7
CILIUM_IMAGE_REPO = quay.io/cilium/cilium

Expand Down Expand Up @@ -531,6 +541,35 @@ kind-install-multus:
kubectl apply -f "$(MULTUS_YAML)"
kubectl -n kube-system rollout status ds kube-multus-ds

.PHONY: kind-install-kubevirt
kind-install-kubevirt: kind-load-image kind-untaint-control-plane
$(call docker_ensure_image_exists,$(KUBEVIRT_OPERATOR_IMAGE))
$(call kind_load_image,kube-ovn,$(KUBEVIRT_OPERATOR_IMAGE))
$(call docker_ensure_image_exists,$(KUBEVIRT_API_IMAGE))
$(call kind_load_image,kube-ovn,$(KUBEVIRT_API_IMAGE))
$(call docker_ensure_image_exists,$(KUBEVIRT_CONTROLLER_IMAGE))
$(call kind_load_image,kube-ovn,$(KUBEVIRT_CONTROLLER_IMAGE))
$(call docker_ensure_image_exists,$(KUBEVIRT_HANDLER_IMAGE))
$(call kind_load_image,kube-ovn,$(KUBEVIRT_HANDLER_IMAGE))
$(call docker_ensure_image_exists,$(KUBEVIRT_LAUNCHER_IMAGE))
$(call kind_load_image,kube-ovn,$(KUBEVIRT_LAUNCHER_IMAGE))
$(call docker_ensure_image_exists,$(KUBEVIRT_TEST_IMAGE))
$(call kind_load_image,kube-ovn,$(KUBEVIRT_TEST_IMAGE))

sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install.sh | bash
kubectl describe no

kubectl apply -f "$(KUBEVIRT_OPERATOR_YAML)"
kubectl apply -f "$(KUBEVIRT_CR_YAML)"
kubectl rollout status deployment/virt-operator -n kubevirt
echo "wait kubevirt releated pod running ..."
sleep 60

kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
kubectl apply -f "$(KUBEVIRT_TEST_YAML)"
sleep 5
kubectl patch vm testvm --type=merge --patch '{"spec":{"running":true}}'

.PHONY: kind-install-lb-svc
kind-install-lb-svc: kind-load-image kind-untaint-control-plane
$(call kind_load_image,kube-ovn,$(VPC_NAT_GW_IMG))
Expand Down
10 changes: 10 additions & 0 deletions Makefile.e2e
Expand Up @@ -59,6 +59,7 @@ e2e-build:
ginkgo build ./test/e2e/lb-svc
ginkgo build ./test/e2e/ovn-eip
ginkgo build ./test/e2e/security
ginkgo build ./test/e2e/kubevirt

.PHONY: k8s-conformance-e2e
k8s-conformance-e2e:
Expand Down Expand Up @@ -160,3 +161,12 @@ kube-ovn-security-e2e:
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \
--focus=CNI:Kube-OVN ./test/e2e/security/security.test

.PHONY: kube-ovn-kubevirt-e2e
kube-ovn-kubevirt-e2e:
ginkgo build ./test/e2e/kubevirt
E2E_BRANCH=$(E2E_BRANCH) \
E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \
--focus=CNI:Kube-OVN ./test/e2e/kubevirt/kubevirt.test
90 changes: 90 additions & 0 deletions test/e2e/kubevirt/e2e_test.go
@@ -0,0 +1,90 @@
package kubevirt

import (
"context"
"flag"
"os"
"path/filepath"
"testing"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
"k8s.io/kubernetes/test/e2e"
k8sframework "k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/config"

"github.com/onsi/ginkgo/v2"

"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/test/e2e/framework"
)

func init() {
klog.SetOutput(ginkgo.GinkgoWriter)

// Register flags.
config.CopyFlags(config.Flags, flag.CommandLine)
k8sframework.RegisterCommonFlags(flag.CommandLine)
k8sframework.RegisterClusterFlags(flag.CommandLine)
}

func TestE2E(t *testing.T) {
if k8sframework.TestContext.KubeConfig == "" {
k8sframework.TestContext.KubeConfig = filepath.Join(os.Getenv("HOME"), ".kube", "config")
}
k8sframework.AfterReadingAllFlags(&k8sframework.TestContext)

e2e.RunE2ETests(t)
}

var _ = framework.Describe("[group:kubevirt]", func() {
f := framework.NewDefaultFramework("kubevirt")

var cs clientset.Interface

ginkgo.BeforeEach(func() {
cs = f.ClientSet
})

framework.ConformanceIt("Kubevirt vm pod should keep ip", func() {
f.SkipVersionPriorTo(1, 12, "Only test kubevirt vm keep ip in master")

ginkgo.By("Get kubevirt vm pod")
podList, err := cs.CoreV1().Pods("default").List(context.TODO(), metav1.ListOptions{
LabelSelector: "vm.kubevirt.io/name=testvm",
})
framework.ExpectNoError(err)
framework.ExpectEqual(len(podList.Items), 1)

ginkgo.By("Validating pod annotations")
pod := podList.Items[0]
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, "ovn.kubernetes.io/virtualmachine", "testvm")
ipAddr := pod.Annotations[util.IpAddressAnnotation]

ginkgo.By("Deleting pod " + pod.Name)
err = cs.CoreV1().Pods("default").Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
framework.ExpectNoError(err)
time.Sleep(10 * time.Second)

ginkgo.By("Check kubevirt vm pod after rebuild")
podList, err = cs.CoreV1().Pods("default").List(context.TODO(), metav1.ListOptions{
LabelSelector: "vm.kubevirt.io/name=testvm",
})
framework.ExpectNoError(err)
framework.ExpectEqual(len(podList.Items), 1)

ginkgo.By("Validating new pod annotations")
pod = podList.Items[0]
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, "ovn.kubernetes.io/virtualmachine", "testvm")

ginkgo.By("Check vm pod ip unchanged" + pod.Name)
ipNewAddr := pod.Annotations[util.IpAddressAnnotation]
framework.ExpectEqual(ipAddr, ipNewAddr)
})
})

0 comments on commit c84479b

Please sign in to comment.