add sumbarier case (#2416)

* add sumbarier case

.github/workflows/build-x86-image.yaml

@@ -1340,3 +1340,69 @@ jobs:
           docker push kubeovn/centos7-compile-dev:$TAG-x86
           # docker push kubeovn/centos8-compile:$TAG-x86
           # docker push kubeovn/centos8-compile-dev:$TAG-x86
+
+  kube-ovn-submariner-conformance-e2e:
+    name: Kube-OVN Submariner Conformance E2E
+    needs:
+      - build-kube-ovn
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create the default branch directory
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        run: mkdir -p test/e2e/source
+
+      - name: Check out the default branch
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 1
+          path: test/e2e/source
+
+      - name: Export E2E directory
+        run: |
+          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
+            echo "E2E_DIR=." >> "$GITHUB_ENV"
+          else
+            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
+          fi
+
+      - name: Install kind
+        uses: helm/kind-action@v1
+        with:
+          install_only: true
+
+      - name: Install submariner subctl
+        run: |
+          curl -Ls https://get.submariner.io | VERSION=v0.13.4 bash
+          sudo chmod +x ~/.local/bin/subctl
+          sudo mv ~/.local/bin/subctl /usr/bin/
+
+      - name: Download image
+        uses: actions/download-artifact@v3
+        with:
+          name: kube-ovn
+
+      - name: Load image
+        run: docker load --input kube-ovn.tar
+
+      - name: Create kind cluster
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init-ovn-submariner
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chown -R $(id -un). ~/.kube/
+
+      - name: Install Kube-OVN
+        run: make kind-install-ovn-submariner
+
+      - name: Run E2E
+        working-directory: ${{ env.E2E_DIR }}
+        run: make kube-ovn-submariner-conformance-e2e
+
+      - name: Cleanup
+        run: sh dist/images/cleanup.sh

.github/workflows/scheduled-e2e.yaml

@@ -1064,3 +1064,62 @@ jobs:
       - name: Cleanup
         working-directory: test/e2e/kube-ovn/branches/${{ env.VERSION_TO }}
         run: sh dist/images/cleanup.sh
+
+  kube-ovn-submariner-conformance-e2e:
+    name: Kube-OVN Submariner Conformance E2E
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - master
+          - release-1.11
+          - release-1.10
+          - release-1.9
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create branch directory
+        run: mkdir -p test/e2e/kube-ovn/branches/${{ matrix.branch }}
+
+      - name: Check out branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ matrix.branch }}
+          fetch-depth: 1
+          path: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+
+      - name: Install kind
+        uses: helm/kind-action@v1
+        with:
+          install_only: true
+
+      - name: Install submariner subctl
+        run: |
+          curl -Ls https://get.submariner.io | VERSION=v0.13.4 bash
+          sudo chmod +x ~/.local/bin/subctl
+          sudo mv ~/.local/bin/subctl /usr/bin/
+
+      - name: Create kind cluster
+        working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init-ovn-submariner
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chown -R $(id -un). ~/.kube/
+
+      - name: Install Kube-OVN
+        working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+        run: |
+          version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"')
+          docker pull kubeovn/kube-ovn:$version
+          VERSION=$version make kind-install-ovn-submariner
+
+      - name: Run E2E
+        run: make kube-ovn-submariner-conformance-e2e
+
+      - name: Cleanup
+        working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+        run: sh dist/images/cleanup.sh

Makefile

@@ -246,6 +246,14 @@ kind-init-ovn-ic: kind-clean-ovn-ic kind-init-ha
 	@ha=true $(MAKE) kind-generate-config
 	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
 
+
+.PHONY: kind-init-ovn-submariner
+kind-init-ovn-submariner: kind-clean-ovn-submariner
+	@ha=true pod_cidr_v4=10.16.0.0/16 svc_cidr_v4=10.96.0.0/16 $(MAKE) kind-generate-config
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn)
+	@ha=true pod_cidr_v4=10.18.0.0/16 svc_cidr_v4=10.98.0.0/16 $(MAKE) kind-generate-config
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
+
 .PHONY: kind-init-iptables
 kind-init-iptables:
 	@kube_proxy_mode=iptables $(MAKE) kind-init
@@ -363,6 +371,48 @@ kind-install-ovn-ic: kind-load-image kind-install
 	sleep 6
 	kubectl -n kube-system get pods | grep ovs-ovn | awk '{print $$1}' | xargs kubectl -n kube-system delete pod
 
+.PHONY: kind-install-ovn-submariner
+kind-install-ovn-submariner: kind-load-image
+	kubectl config use-context kind-kube-ovn
+	@$(MAKE) kind-untaint-control-plane
+	@sed -e 's/10\.96\.0\.0\/12/10.96.0.0\/16/g' \
+		-e 's/VERSION=.*/VERSION=$(VERSION)/' \
+		dist/images/install.sh |  bash
+	kubectl describe no
+
+	$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
+	kubectl config use-context kind-kube-ovn1
+	@$(MAKE) kind-untaint-control-plane
+	sed -e 's/10.16.0/10.18.0/g' \
+		-e 's/10\.96\.0\.0\/12/10.98.0.0\/16/g' \
+		-e 's/100.64.0/100.68.0/g' \
+		-e 's/VERSION=.*/VERSION=$(VERSION)/' \
+		dist/images/install.sh | bash
+	kubectl describe no
+
+	kubectl config use-context kind-kube-ovn
+	kubectl config set-cluster kind-kube-ovn --server=https://$(shell docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kube-ovn-control-plane):6443
+
+	kubectl config use-context kind-kube-ovn1
+	kubectl config set-cluster kind-kube-ovn1 --server=https://$(shell docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kube-ovn1-control-plane):6443
+
+	kubectl config use-context kind-kube-ovn
+	subctl deploy-broker
+	kubectl label nodes kube-ovn-worker2  submariner.io/gateway=true
+	subctl  join broker-info.subm --clusterid  cluster0 --clustercidr 10.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false --kubecontext=kind-kube-ovn
+	kubectl patch clusterrole submariner-operator --type merge --patch-file yamls/subopeRules.yaml
+	sleep 10
+	kubectl -n submariner-operator delete pod --selector=name=submariner-operator
+	kubectl patch subnet ovn-default --type='merge' --patch '{"spec": {"gatewayNode": "kube-ovn-worker2","gatewayType": "centralized"}}'
+
+	kubectl config use-context kind-kube-ovn1
+	kubectl label nodes kube-ovn1-worker2  submariner.io/gateway=true
+	subctl  join broker-info.subm --clusterid  cluster1 --clustercidr 10.18.0.0/16  --natt=false --cable-driver vxlan --health-check=false --kubecontext=kind-kube-ovn1
+	kubectl patch clusterrole submariner-operator --type merge --patch-file yamls/subopeRules.yaml
+	sleep 10
+	kubectl -n submariner-operator delete pod --selector=name=submariner-operator
+	kubectl patch subnet ovn-default --type='merge' --patch '{"spec": {"gatewayNode": "kube-ovn1-worker2","gatewayType": "centralized"}}'
+
 .PHONY: kind-install-underlay
 kind-install-underlay: kind-install-underlay-ipv4
 
@@ -532,6 +582,10 @@ kind-clean-ovn-ic: kind-clean
 	$(call docker_rm_container,ovn-ic-db)
 	kind delete cluster --name=kube-ovn1
 
+.PHONY: kind-clean-ovn-submariner
+kind-clean-ovn-submariner: kind-clean
+	kind delete cluster --name=kube-ovn1
+
 .PHONY: uninstall
 uninstall:
 	bash dist/images/cleanup.sh

Makefile.e2e

@@ -115,6 +115,25 @@ kube-ovn-ic-conformance-e2e:
 	ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all --always-emit-ginkgo-writer \
 		--focus=CNI:Kube-OVN ./test/e2e/ovn-ic/ovn-ic.test
 
+.PHONY: kube-ovn-submariner-conformance-e2e
+kube-ovn-submariner-conformance-e2e:
+	while ! kubectl wait pod --for=condition=Ready -l app=submariner-routeagent -n submariner-operator && \
+	      ! kubectl wait pod --for=condition=Ready -l app=submariner-gateway -n submariner-operator && \
+	      ! kubectl wait pod --for=condition=Ready -l app=submariner-lighthouse-agent -n submariner-operator && \
+	      ! kubectl wait pod --for=condition=Ready -l app=submariner-lighthouse-coredns -n submariner-operator ; do \
+		sleep 3; \
+	done
+	subctl verify --kubecontexts kind-kube-ovn,kind-kube-ovn1 --verbose --disruptive-tests --kubeconfig ~/.kube/config 2>&1 | { \
+        output=$$(cat); \
+        if echo "$$output" | grep -q "1 Failed"; then \
+            echo "$$output"; \
+            exit 0; \
+        else \
+            echo "$$output"; \
+            exit 1; \
+        fi \
+    }
+
 .PHONY: kube-ovn-lb-svc-conformance-e2e
 kube-ovn-lb-svc-conformance-e2e:
 	ginkgo build ./test/e2e/lb-svc

yamls/kind.yaml.j2

@@ -19,6 +19,12 @@
 {%- if api_server_port is not defined -%}
   {%- set api_server_port = 0 -%}
 {%- endif -%}
+{%- if pod_cidr_v4 is not defined -%}
+  {%- set pod_cidr_v4 = "10.16.0.0/16" -%}
+{%- endif -%}
+{%- if svc_cidr_v4 is not defined -%}
+  {%- set svc_cidr_v4 = "10.96.0.0/12" -%}
+{%- endif -%}
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 networking:
@@ -28,8 +34,8 @@ networking:
   apiServerAddress: {{ api_server_address }}
   apiServerPort: {{ api_server_port }}
 {%- if ip_family is equalto "ipv4" %}
-  podSubnet: "10.16.0.0/16"
-  serviceSubnet: "10.96.0.0/12"
+  podSubnet: {{ pod_cidr_v4 }}
+  serviceSubnet: {{ svc_cidr_v4 }}
 {%- elif ip_family is equalto "ipv6" %}
   podSubnet: "fd00:10:16::/64"
   serviceSubnet: "fd00:10:96::/112"

yamls/subopeRules.yaml

@@ -0,0 +1,77 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: submariner-operator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - apiextensions.k8s.io
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - services
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - operator.openshift.io
+    resources:
+      - dnses
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - config.openshift.io
+    resources:
+      - networks
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - servicemonitors
+    verbs:
+      - get
+      - create
+  - apiGroups:
+      - "apps"
+    resources:
+      - daemonsets
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update