From 25fef7cceea1a715369da5e97a61489a4e6099c4 Mon Sep 17 00:00:00 2001 From: changluyi <47097611+changluyi@users.noreply.github.com> Date: Tue, 7 Mar 2023 11:30:32 +0800 Subject: [PATCH] add sumbarier case (#2416) * add sumbarier case --- .github/workflows/build-x86-image.yaml | 66 ++++++++++++++++++++++ .github/workflows/scheduled-e2e.yaml | 59 ++++++++++++++++++++ Makefile | 54 ++++++++++++++++++ Makefile.e2e | 19 +++++++ yamls/kind.yaml.j2 | 10 +++- yamls/subopeRules.yaml | 77 ++++++++++++++++++++++++++ 6 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 yamls/subopeRules.yaml diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index f877bbec599..0181119f45c 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -1340,3 +1340,69 @@ jobs: docker push kubeovn/centos7-compile-dev:$TAG-x86 # docker push kubeovn/centos8-compile:$TAG-x86 # docker push kubeovn/centos8-compile-dev:$TAG-x86 + + kube-ovn-submariner-conformance-e2e: + name: Kube-OVN Submariner Conformance E2E + needs: + - build-kube-ovn + runs-on: ubuntu-20.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + + - name: Create the default branch directory + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + run: mkdir -p test/e2e/source + + - name: Check out the default branch + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + uses: actions/checkout@v3 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + path: test/e2e/source + + - name: Export E2E directory + run: | + if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then + echo "E2E_DIR=." >> "$GITHUB_ENV" + else + echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" + fi + + - name: Install kind + uses: helm/kind-action@v1 + with: + install_only: true + + - name: Install submariner subctl + run: | + curl -Ls https://get.submariner.io | VERSION=v0.13.4 bash + sudo chmod +x ~/.local/bin/subctl + sudo mv ~/.local/bin/subctl /usr/bin/ + + - name: Download image + uses: actions/download-artifact@v3 + with: + name: kube-ovn + + - name: Load image + run: docker load --input kube-ovn.tar + + - name: Create kind cluster + run: | + sudo pip3 install j2cli + sudo pip3 install "j2cli[yaml]" + sudo PATH=~/.local/bin:$PATH make kind-init-ovn-submariner + sudo cp -r /root/.kube/ ~/.kube/ + sudo chown -R $(id -un). ~/.kube/ + + - name: Install Kube-OVN + run: make kind-install-ovn-submariner + + - name: Run E2E + working-directory: ${{ env.E2E_DIR }} + run: make kube-ovn-submariner-conformance-e2e + + - name: Cleanup + run: sh dist/images/cleanup.sh \ No newline at end of file diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml index fcef4bee4cc..fb27cc6265f 100644 --- a/.github/workflows/scheduled-e2e.yaml +++ b/.github/workflows/scheduled-e2e.yaml @@ -1064,3 +1064,62 @@ jobs: - name: Cleanup working-directory: test/e2e/kube-ovn/branches/${{ env.VERSION_TO }} run: sh dist/images/cleanup.sh + + kube-ovn-submariner-conformance-e2e: + name: Kube-OVN Submariner Conformance E2E + runs-on: ubuntu-20.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + branch: + - master + - release-1.11 + - release-1.10 + - release-1.9 + steps: + - uses: actions/checkout@v3 + + - name: Create branch directory + run: mkdir -p test/e2e/kube-ovn/branches/${{ matrix.branch }} + + - name: Check out branch + uses: actions/checkout@v3 + with: + ref: ${{ matrix.branch }} + fetch-depth: 1 + path: test/e2e/kube-ovn/branches/${{ matrix.branch }} + + - name: Install kind + uses: helm/kind-action@v1 + with: + install_only: true + + - name: Install submariner subctl + run: | + curl -Ls https://get.submariner.io | VERSION=v0.13.4 bash + sudo chmod +x ~/.local/bin/subctl + sudo mv ~/.local/bin/subctl /usr/bin/ + + - name: Create kind cluster + working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }} + run: | + sudo pip3 install j2cli + sudo pip3 install "j2cli[yaml]" + sudo PATH=~/.local/bin:$PATH make kind-init-ovn-submariner + sudo cp -r /root/.kube/ ~/.kube/ + sudo chown -R $(id -un). ~/.kube/ + + - name: Install Kube-OVN + working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }} + run: | + version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"') + docker pull kubeovn/kube-ovn:$version + VERSION=$version make kind-install-ovn-submariner + + - name: Run E2E + run: make kube-ovn-submariner-conformance-e2e + + - name: Cleanup + working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }} + run: sh dist/images/cleanup.sh \ No newline at end of file diff --git a/Makefile b/Makefile index 8ccaa68e480..e79a5f96d4d 100644 --- a/Makefile +++ b/Makefile @@ -246,6 +246,14 @@ kind-init-ovn-ic: kind-clean-ovn-ic kind-init-ha @ha=true $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1) + +.PHONY: kind-init-ovn-submariner +kind-init-ovn-submariner: kind-clean-ovn-submariner + @ha=true pod_cidr_v4=10.16.0.0/16 svc_cidr_v4=10.96.0.0/16 $(MAKE) kind-generate-config + $(call kind_create_cluster,yamls/kind.yaml,kube-ovn) + @ha=true pod_cidr_v4=10.18.0.0/16 svc_cidr_v4=10.98.0.0/16 $(MAKE) kind-generate-config + $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1) + .PHONY: kind-init-iptables kind-init-iptables: @kube_proxy_mode=iptables $(MAKE) kind-init @@ -363,6 +371,48 @@ kind-install-ovn-ic: kind-load-image kind-install sleep 6 kubectl -n kube-system get pods | grep ovs-ovn | awk '{print $$1}' | xargs kubectl -n kube-system delete pod +.PHONY: kind-install-ovn-submariner +kind-install-ovn-submariner: kind-load-image + kubectl config use-context kind-kube-ovn + @$(MAKE) kind-untaint-control-plane + @sed -e 's/10\.96\.0\.0\/12/10.96.0.0\/16/g' \ + -e 's/VERSION=.*/VERSION=$(VERSION)/' \ + dist/images/install.sh | bash + kubectl describe no + + $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) + kubectl config use-context kind-kube-ovn1 + @$(MAKE) kind-untaint-control-plane + sed -e 's/10.16.0/10.18.0/g' \ + -e 's/10\.96\.0\.0\/12/10.98.0.0\/16/g' \ + -e 's/100.64.0/100.68.0/g' \ + -e 's/VERSION=.*/VERSION=$(VERSION)/' \ + dist/images/install.sh | bash + kubectl describe no + + kubectl config use-context kind-kube-ovn + kubectl config set-cluster kind-kube-ovn --server=https://$(shell docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kube-ovn-control-plane):6443 + + kubectl config use-context kind-kube-ovn1 + kubectl config set-cluster kind-kube-ovn1 --server=https://$(shell docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kube-ovn1-control-plane):6443 + + kubectl config use-context kind-kube-ovn + subctl deploy-broker + kubectl label nodes kube-ovn-worker2 submariner.io/gateway=true + subctl join broker-info.subm --clusterid cluster0 --clustercidr 10.16.0.0/16 --natt=false --cable-driver vxlan --health-check=false --kubecontext=kind-kube-ovn + kubectl patch clusterrole submariner-operator --type merge --patch-file yamls/subopeRules.yaml + sleep 10 + kubectl -n submariner-operator delete pod --selector=name=submariner-operator + kubectl patch subnet ovn-default --type='merge' --patch '{"spec": {"gatewayNode": "kube-ovn-worker2","gatewayType": "centralized"}}' + + kubectl config use-context kind-kube-ovn1 + kubectl label nodes kube-ovn1-worker2 submariner.io/gateway=true + subctl join broker-info.subm --clusterid cluster1 --clustercidr 10.18.0.0/16 --natt=false --cable-driver vxlan --health-check=false --kubecontext=kind-kube-ovn1 + kubectl patch clusterrole submariner-operator --type merge --patch-file yamls/subopeRules.yaml + sleep 10 + kubectl -n submariner-operator delete pod --selector=name=submariner-operator + kubectl patch subnet ovn-default --type='merge' --patch '{"spec": {"gatewayNode": "kube-ovn1-worker2","gatewayType": "centralized"}}' + .PHONY: kind-install-underlay kind-install-underlay: kind-install-underlay-ipv4 @@ -532,6 +582,10 @@ kind-clean-ovn-ic: kind-clean $(call docker_rm_container,ovn-ic-db) kind delete cluster --name=kube-ovn1 +.PHONY: kind-clean-ovn-submariner +kind-clean-ovn-submariner: kind-clean + kind delete cluster --name=kube-ovn1 + .PHONY: uninstall uninstall: bash dist/images/cleanup.sh diff --git a/Makefile.e2e b/Makefile.e2e index a739066d65a..19c9ce1acdb 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -115,6 +115,25 @@ kube-ovn-ic-conformance-e2e: ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \ --focus=CNI:Kube-OVN ./test/e2e/ovn-ic/ovn-ic.test +.PHONY: kube-ovn-submariner-conformance-e2e +kube-ovn-submariner-conformance-e2e: + while ! kubectl wait pod --for=condition=Ready -l app=submariner-routeagent -n submariner-operator && \ + ! kubectl wait pod --for=condition=Ready -l app=submariner-gateway -n submariner-operator && \ + ! kubectl wait pod --for=condition=Ready -l app=submariner-lighthouse-agent -n submariner-operator && \ + ! kubectl wait pod --for=condition=Ready -l app=submariner-lighthouse-coredns -n submariner-operator ; do \ + sleep 3; \ + done + subctl verify --kubecontexts kind-kube-ovn,kind-kube-ovn1 --verbose --disruptive-tests --kubeconfig ~/.kube/config 2>&1 | { \ + output=$$(cat); \ + if echo "$$output" | grep -q "1 Failed"; then \ + echo "$$output"; \ + exit 0; \ + else \ + echo "$$output"; \ + exit 1; \ + fi \ + } + .PHONY: kube-ovn-lb-svc-conformance-e2e kube-ovn-lb-svc-conformance-e2e: ginkgo build ./test/e2e/lb-svc diff --git a/yamls/kind.yaml.j2 b/yamls/kind.yaml.j2 index f90679571f3..0de39f407d6 100644 --- a/yamls/kind.yaml.j2 +++ b/yamls/kind.yaml.j2 @@ -19,6 +19,12 @@ {%- if api_server_port is not defined -%} {%- set api_server_port = 0 -%} {%- endif -%} +{%- if pod_cidr_v4 is not defined -%} + {%- set pod_cidr_v4 = "10.16.0.0/16" -%} +{%- endif -%} +{%- if svc_cidr_v4 is not defined -%} + {%- set svc_cidr_v4 = "10.96.0.0/12" -%} +{%- endif -%} kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: @@ -28,8 +34,8 @@ networking: apiServerAddress: {{ api_server_address }} apiServerPort: {{ api_server_port }} {%- if ip_family is equalto "ipv4" %} - podSubnet: "10.16.0.0/16" - serviceSubnet: "10.96.0.0/12" + podSubnet: {{ pod_cidr_v4 }} + serviceSubnet: {{ svc_cidr_v4 }} {%- elif ip_family is equalto "ipv6" %} podSubnet: "fd00:10:16::/64" serviceSubnet: "fd00:10:96::/112" diff --git a/yamls/subopeRules.yaml b/yamls/subopeRules.yaml new file mode 100644 index 00000000000..c9dcb1a56a7 --- /dev/null +++ b/yamls/subopeRules.yaml @@ -0,0 +1,77 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: submariner-operator +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - "" + resources: + - pods + - services + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - operator.openshift.io + resources: + - dnses + verbs: + - get + - list + - watch + - update + - apiGroups: + - config.openshift.io + resources: + - networks + verbs: + - get + - list + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - apiGroups: + - "apps" + resources: + - daemonsets + verbs: + - create + - get + - list + - watch + - update