Split e2e test cases (#949)

.github/actions/e2e/action.yaml

@@ -0,0 +1,68 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Run E2E tests
+description: "Run an E2E test suite"
+
+inputs:
+  testsuite_name:
+    description: "Name of the test suite to run"
+    required: true
+  concurrency:
+    description: "Number of concurrent test runs"
+    required: false
+    default: '2'
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install kind
+      shell: bash
+      run: |
+        curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
+        chmod +x ./kind
+        sudo mv kind /usr/local/bin
+
+    - name: Setup Go Env
+      uses: actions/setup-go@v2
+      with:
+        go-version: "1.16"
+
+    - name: Install ginkgo
+      shell: bash
+      run: |
+        go install github.com/onsi/ginkgo/ginkgo@v1.16.4
+        sudo cp ~/go/bin/ginkgo /usr/local/bin
+
+    - name: Create K8s cluster
+      shell: bash
+      run: |
+        make kind-up
+        kubectl wait --for=condition=Ready nodes --all
+
+    - name: Run E2E test suite
+      shell: bash
+      env:
+        E2E_FOCUS: "${{ inputs.testsuite_name }}*"
+        E2E_CONCURRENCY: "${{ inputs.concurrency }}"
+      run: |
+        make e2e-test

.github/workflows/e2e-test-ci.yml

@@ -62,37 +62,33 @@ jobs:
               - 'conf/**'
               - 'utils/**'
 
-  e2e-test:
+  prepare-matrix:
     needs: changes
-    if: |
-      (needs.changes.outputs.go == 'true')
+    if: needs.changes.outputs.go == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: recursive
-      - name: Install kind
+
+      - name: List test suites and set the matrix
+        id: set-matrix
         run: |
-          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
-          chmod +x ./kind
-          sudo mv kind /usr/local/bin
-      - name: Setup Go Env
-        uses: actions/setup-go@v2
+          SUITES=($(find test/e2e -type d -iname 'suite-*' | grep -E -o '(\w|\-)*' | grep -v 'test' | grep -v 'e2e'))
+          echo $SUITES
+          echo "::set-output name=matrix::$(jq --compact-output --null-input '$ARGS.positional' --args "${SUITES[@]}")"
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+  e2e-test:
+    needs: prepare-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        suite: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/e2e
         with:
-          go-version: "1.16"
-      - name: Install ginkgo
-        run: |
-          go install github.com/onsi/ginkgo/ginkgo@v1.16.4
-          sudo cp ~/go/bin/ginkgo /usr/local/bin
-      - name: Create K8s cluster
-        run: |
-          make kind-up
-          kubectl wait --for=condition=Ready nodes --all
-      - name: Run e2e test cases
-        working-directory: ./
-        run: |
-          make e2e-test E2E_CONCURRENCY=2
-      - name: upload coverage profile
-        working-directory: ./test/e2e
-        run: |
-          bash <(curl -s https://codecov.io/bash)
+          testsuite_name: ${{ matrix.suite }}

Makefile

@@ -70,7 +70,7 @@ e2e-test: ginkgo-check push-images
 	cd test/e2e \
 		&& go mod download \
 		&& export REGISTRY=$(REGISTRY) \
-		&& ACK_GINKGO_RC=true ginkgo -cover -coverprofile=coverage.txt -r --randomizeSuites --randomizeAllSpecs --trace --nodes=$(E2E_CONCURRENCY)
+		&& ACK_GINKGO_RC=true ginkgo -cover -coverprofile=coverage.txt -r --randomizeSuites --randomizeAllSpecs --trace --nodes=$(E2E_CONCURRENCY) --focus=$(E2E_FOCUS)
 
 ### e2e-test-local:        Run e2e test cases (kind is required)
 .PHONY: e2e-test-local

test/e2e/README.md

@@ -41,7 +41,7 @@ Test cases inside `plugins` directory test the availability about APISIX plugins
 Features
 --------
 
-Test caes inside `features` directory test some features about APISIX, such as traffic-split, health check and so on.
+Test case inside `features` directory test some features about APISIX, such as traffic-split, health check and so on.
 
 Quick Start
 -----------
@@ -57,4 +57,15 @@ Run `make e2e-test` to run the e2e test suites in an existing cluster, you can s
 Step `1` and `2` can be skipped by passing `E2E_SKIP_BUILD=1` to this directive, also, you can customize the
 running concurrency of e2e test suites by passing `E2E_CONCURRENCY=X` where `X` is the desired number of cases running in parallel.
 
+You can run specific test cases by passing the environment variable `E2E_FOCUS=suite-<suite name>`, where `<suite name>` can be found under `test/e2e` directory.
+For example, `E2E_FOCUS=suite-plugins* make e2e-test` will only run test cases in `test/e2e/suite-plugins` directory.
+
 Run `make kind-reset` to delete the cluster that created by `make e2e-test`.
+
+How to name test cases
+-----------
+
+Because we use `ginkgo --focus` option and the prefix `suite-<suite name>` to split test cases and make them run in parallel in CI, test cases should be named in the following way:
+
+- All test cases are grouped by directories, and **their names should have `suite-` prefix**
+- All top level specs (i.e. `ginkgo.Describe`) under the suite directory should have corresponding `suite-<suite-name>: ` prefix.

test/e2e/e2e.go

@@ -15,13 +15,13 @@
 package e2e
 
 import (
-	_ "github.com/apache/apisix-ingress-controller/test/e2e/annotations"
-	_ "github.com/apache/apisix-ingress-controller/test/e2e/chaos"
-	_ "github.com/apache/apisix-ingress-controller/test/e2e/config"
-	_ "github.com/apache/apisix-ingress-controller/test/e2e/endpoints"
-	_ "github.com/apache/apisix-ingress-controller/test/e2e/features"
-	_ "github.com/apache/apisix-ingress-controller/test/e2e/ingress"
-	_ "github.com/apache/apisix-ingress-controller/test/e2e/plugins"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-annotations"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-chaos"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-config"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-endpoints"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-features"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-ingress"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/suite-plugins"
 )
 
 func runE2E() {}

test/e2e/annotations/cors.go → test/e2e/suite-annotations/cors.go

@@ -25,7 +25,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("cors annotations", func() {
+var _ = ginkgo.Describe("suite-annotations: cors annotations", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("enable in ingress networking/v1", func() {
@@ -61,14 +61,14 @@ spec:
 		resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 
 		resp = s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").WithHeader("Origin", "https://baz.com").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 
@@ -112,7 +112,7 @@ spec:
 		resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").WithHeader("Origin", "https://foo.com").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 	})
@@ -148,14 +148,14 @@ spec:
 		resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 
 		resp = s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").WithHeader("Origin", "https://baz.com").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 
@@ -197,14 +197,14 @@ spec:
 		resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 
 		resp = s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").WithHeader("Origin", "https://foo.com").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 	})
@@ -240,14 +240,14 @@ spec:
 		resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 
 		resp = s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").WithHeader("Origin", "https://baz.com").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 
@@ -289,7 +289,7 @@ spec:
 		resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").WithHeader("Origin", "https://foo.com").Expect()
 		resp.Status(http.StatusOK)
 		// As httpbin itself adds this header, we don't check it here.
-		//resp.Header("Access-Control-Allow-Origin").Empty()
+		// resp.Header("Access-Control-Allow-Origin").Empty()
 		resp.Header("Access-Control-Allow-Methods").Empty()
 		resp.Header("Access-Control-Allow-Headers").Empty()
 	})

test/e2e/annotations/iprestriction.go → test/e2e/suite-annotations/iprestriction.go

@@ -25,7 +25,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("allowlist-source-range annotations", func() {
+var _ = ginkgo.Describe("suite-annotations: allowlist-source-range annotations", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("enable in ingress networking/v1", func() {

test/e2e/annotations/redirect.go → test/e2e/suite-annotations/redirect.go

@@ -25,7 +25,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("redirect annotations", func() {
+var _ = ginkgo.Describe("suite-annotations: redirect annotations", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("redirect http-to-https in ingress networking/v1", func() {

test/e2e/annotations/rewrite.go → test/e2e/suite-annotations/rewrite.go

@@ -25,7 +25,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("rewrite annotations", func() {
+var _ = ginkgo.Describe("suite-annotations: rewrite annotations", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("enable in ingress networking/v1", func() {

test/e2e/chaos/chaos.go → test/e2e/suite-chaos/chaos.go

@@ -24,7 +24,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("Chaos Testing", func() {
+var _ = ginkgo.Describe("suite-chaos: Chaos Testing", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/config/config.go → test/e2e/suite-config/config.go

@@ -27,7 +27,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("deploy ingress controller with config", func() {
+var _ = ginkgo.Describe("suite-config: deploy ingress controller with config", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/endpoints/endpoints.go → test/e2e/suite-endpoints/endpoints.go

@@ -25,7 +25,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("endpoints", func() {
+var _ = ginkgo.Describe("suite-endpoints: endpoints", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/features/consumer.go → test/e2e/suite-features/consumer.go

@@ -25,7 +25,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("ApisixConsumer", func() {
+var _ = ginkgo.Describe("suite-features: ApisixConsumer", func() {
 	s := scaffold.NewDefaultV2Scaffold()
 
 	ginkgo.It("ApisixRoute with basicAuth consumer", func() {

test/e2e/features/global_rule.go → test/e2e/suite-features/global_rule.go

@@ -19,14 +19,14 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/apache/apisix-ingress-controller/pkg/id"
 	"github.com/onsi/ginkgo"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/apache/apisix-ingress-controller/pkg/id"
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("ApisixClusterConfig", func() {
+var _ = ginkgo.Describe("suite-features: ApisixClusterConfig", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/features/healthcheck.go → test/e2e/suite-features/healthcheck.go

@@ -24,7 +24,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("health check", func() {
+var _ = ginkgo.Describe("suite-features: health check", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/features/remote_addrs_match.go → test/e2e/suite-features/remote_addrs_match.go

@@ -24,7 +24,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("traffic split", func() {
+var _ = ginkgo.Describe("suite-features: traffic split", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/features/retries.go → test/e2e/suite-features/retries.go

@@ -24,7 +24,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("retries", func() {
+var _ = ginkgo.Describe("suite-features: retries", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/features/route_match_exprs.go → test/e2e/suite-features/route_match_exprs.go

@@ -25,7 +25,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
-var _ = ginkgo.Describe("route match exprs", func() {
+var _ = ginkgo.Describe("suite-features: route match exprs", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),