Add a pre-check for E2E tests (#957)

.github/actions/e2e/action.yaml

@@ -64,5 +64,6 @@ runs:
       env:
         E2E_FOCUS: "${{ inputs.testsuite_name }}*"
         E2E_CONCURRENCY: "${{ inputs.concurrency }}"
+        ENABLE_PROXY: "false"
       run: |
         make e2e-test

.github/workflows/e2e-test-ci.yml

@@ -62,7 +62,7 @@ jobs:
               - 'conf/**'
               - 'utils/**'
 
-  prepare-matrix:
+  prepare:
     needs: changes
     if: needs.changes.outputs.go == 'true'
     runs-on: ubuntu-latest
@@ -71,6 +71,9 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Check e2e test cases' naming
+        run: make e2e-names-check
+
       - name: List test suites and set the matrix
         id: set-matrix
         run: |
@@ -81,11 +84,12 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
 
   e2e-test:
-    needs: prepare-matrix
+    needs: prepare
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false # If false, GitHub will not cancels all in-progress jobs in the matrix if any matrix job fails.
       matrix:
-        suite: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
+        suite: ${{ fromJson(needs.prepare.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v2
 

Makefile

@@ -20,6 +20,7 @@ VERSION ?= 1.4.0
 RELEASE_SRC = apache-apisix-ingress-controller-${VERSION}-src
 REGISTRY ?="localhost:5000"
 IMAGE_TAG ?= dev
+ENABLE_PROXY ?= true
 
 GITSHA ?= "no-git-module"
 ifneq ("$(wildcard .git)", "")
@@ -99,11 +100,11 @@ ifeq ($(E2E_SKIP_BUILD), 0)
 	docker tag kennethreitz/httpbin $(REGISTRY)/kennethreitz/httpbin
 	docker push $(REGISTRY)/kennethreitz/httpbin
 
-	docker build -t test-backend:$(IMAGE_TAG) --build-arg ENABLE_PROXY=true ./test/e2e/testbackend
+	docker build -t test-backend:$(IMAGE_TAG) --build-arg ENABLE_PROXY=$(ENABLE_PROXY) ./test/e2e/testbackend
 	docker tag test-backend:$(IMAGE_TAG) $(REGISTRY)/test-backend:$(IMAGE_TAG)
 	docker push $(REGISTRY)/test-backend:$(IMAGE_TAG)
 
-	docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) --build-arg ENABLE_PROXY=true .
+	docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) --build-arg ENABLE_PROXY=$(ENABLE_PROXY) .
 	docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
 	docker push $(REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
 
@@ -203,3 +204,8 @@ update-gofmt:
 ### update-all:           Update all update- rules.
 .PHONY: update-all
 update-all: update-codegen update-license update-mdlint update-gofmt
+
+### e2e-names-check:          Check if e2e test cases' names have the prefix "suite-<suite-name>".
+.PHONY: e2e-names-check
+e2e-names-check:
+	chmod +x ./utils/check-e2e-names.sh && ./utils/check-e2e-names.sh

test/e2e/README.md

@@ -69,3 +69,5 @@ Because we use `ginkgo --focus` option and the prefix `suite-<suite name>` to sp
 
 - All test cases are grouped by directories, and **their names should have `suite-` prefix**
 - All top level specs (i.e. `ginkgo.Describe`) under the suite directory should have corresponding `suite-<suite-name>: ` prefix.
+
+Run `make names-check` to check the above naming convention.

test/e2e/suite-annotations/iprestriction.go

@@ -118,7 +118,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("blocklist-source-range annotations", func() {
+var _ = ginkgo.Describe("suite-annotations: blocklist-source-range annotations", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("enable in ingress networking/v1", func() {

test/e2e/suite-annotations/rewrite.go

@@ -118,7 +118,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("rewrite regex annotations", func() {
+var _ = ginkgo.Describe("suite-annotations: rewrite regex annotations", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("enable in ingress networking/v1", func() {

test/e2e/suite-endpoints/endpoints.go

@@ -90,7 +90,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("port usage", func() {
+var _ = ginkgo.Describe("suite-endpoints: port usage", func() {
 	opts := &scaffold.Options{
 		Name:                  "endpoints-port",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/suite-features/retries.go

@@ -127,7 +127,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("retries timeout", func() {
+var _ = ginkgo.Describe("suite-features: retries timeout", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/suite-features/route_match_exprs.go

@@ -602,7 +602,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("route match exprs bugfixes", func() {
+var _ = ginkgo.Describe("suite-features: route match exprs bugfixes", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/suite-ingress/ingress.go

@@ -317,7 +317,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("support ingress.networking/v1", func() {
+var _ = ginkgo.Describe("suite-ingress: support ingress.networking/v1", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("path exact match", func() {
@@ -421,7 +421,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("support ingress.networking/v1beta1", func() {
+var _ = ginkgo.Describe("suite-ingress: support ingress.networking/v1beta1", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("path exact match", func() {
@@ -519,7 +519,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("support ingress.extensions/v1beta1", func() {
+var _ = ginkgo.Describe("suite-ingress: support ingress.extensions/v1beta1", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	ginkgo.It("path exact match", func() {
@@ -617,7 +617,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("support ingress.networking/v1 with headless service backend", func() {
+var _ = ginkgo.Describe("suite-ingress: support ingress.networking/v1 with headless service backend", func() {
 	s := scaffold.NewDefaultScaffold()
 
 	const _httpHeadlessService = `

test/e2e/suite-ingress/sanity.go

@@ -77,7 +77,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("double-routes", func() {
+var _ = ginkgo.Describe("suite-ingress: double-routes", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),
@@ -135,7 +135,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("leader election", func() {
+var _ = ginkgo.Describe("suite-ingress: leader election", func() {
 	s := scaffold.NewScaffold(&scaffold.Options{
 		Name:                  "leaderelection",
 		Kubeconfig:            scaffold.GetKubeconfig(),
@@ -189,7 +189,7 @@ var _ = ginkgo.Describe("leader election", func() {
 	})
 })
 
-var _ = ginkgo.Describe("stream_routes disabled", func() {
+var _ = ginkgo.Describe("suite-ingress: stream_routes disabled", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

test/e2e/suite-ingress/ssl.go

@@ -241,7 +241,7 @@ RU+QPRECgYB6XW24EI5+w3STbpnc6VoTS+sy9I9abTJPYo9LpCJwfMYc9Tg9Cx2K
 	})
 })
 
-var _ = ginkgo.Describe("ApisixTls mTLS Test", func() {
+var _ = ginkgo.Describe("suite-ingress: ApisixTls mTLS Test", func() {
 	// RootCA -> Server
 	// RootCA -> UserCert
 	// These certs come from mTLS practice

test/e2e/suite-ingress/status.go

@@ -72,7 +72,7 @@ spec:
 	})
 })
 
-var _ = ginkgo.Describe("Ingress LB Status Testing", func() {
+var _ = ginkgo.Describe("suite-ingress: Ingress LB Status Testing", func() {
 	opts := &scaffold.Options{
 		Name:                  "default",
 		Kubeconfig:            scaffold.GetKubeconfig(),

utils/check-e2e-names.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Search for the pattern in all files recursively in `test/e2e`, showing line numbers of matches, ignoring binary files.
+# The results are separated by '\n' and stored in the array lines.
+# Each line in lines looks like: `test/e2e/suite-endpoints/endpoints.go:28:var _ = ginkgo.Describe("suite-endpoints: endpoints", func() {`
+IFS=$'\n' lines=($(grep --recursive --line-number --binary-files=without-match "ginkgo.Describe(" test/e2e))
+
+# How many lines do not have the "suite-<suite-name>" prefix.
+err=0
+
+for (( i=0;i<${#lines[@]};i++)); do
+  # Find the second colon in the line to split the line into two parts: left_str and right_str.
+  pos1=$(echo "${lines[i]}" | sed -n "s/[:].*//p" | wc -c | xargs)
+  temp_str=${lines[i]:$pos1}
+  pos2=$(echo "$temp_str" | sed -n "s/[:].*//p" | wc -c | xargs)
+
+  # left_str looks like: `test/e2e/suite-endpoints/endpoints.go:28`
+  left_str=$(echo "${lines[i]}" | cut -c1-$(expr $pos1 + $pos2 - 1))
+  # right_str looks like: `var _ = ginkgo.Describe("suite-endpoints: endpoints", func() {`
+  right_str=${lines[i]:$pos1+$pos2}
+
+  l_name=$(echo "$left_str" | grep --extended-regexp -o 'suite-\w+')
+  r_name=$(echo "$right_str" | grep --extended-regexp -o 'suite-\w+')
+
+  if [ -n "$l_name" ] && [ "$l_name" != "$r_name" ]; then
+    echo "[ERROR]$left_str: $l_name is required"
+    err+=1
+  fi
+done;
+
+if [ $err -gt 0 ]; then
+  echo "-------------------------------------------------------------------------------------"
+  echo 'The prefix "suite-<suite-name>" is required, see test/e2e/README.md for more details.'
+  echo "-------------------------------------------------------------------------------------"
+  exit 1
+fi