feat: add support for e2e tests

Signed-off-by: bitliu <bitliu@tencent.com>

.github/workflows/build-and-test.yml

@@ -7,11 +7,16 @@ on:
     branches: ["*"]
 
 jobs:
-  #TODO(@Xunzhuo): add lint tools to do static code analyse.
   lint:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/setup-go@v3
+      with:
+        go-version: 1.19
+    # There are too many lint errors in current code bases
+    # uncomment when we descide what lint should be addressed or ignored.
+    # - run: make lint
 
   coverage-test:
     runs-on: ubuntu-latest
@@ -32,10 +37,6 @@ jobs:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
     needs: [lint,coverage-test]
-    strategy:
-      matrix:
-        golang:
-          - 1.19
     steps:
       - name: "Setup Go"
         uses: actions/setup-go@v3
@@ -67,7 +68,12 @@ jobs:
     runs-on: ubuntu-latest
     needs: [build]
     steps:
+    - name: "Setup Go"
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19
     - uses: actions/checkout@v3
+    - run: make e2e-test
 
   publish:
     runs-on: ubuntu-latest

.gitignore

@@ -7,4 +7,5 @@ out
 bazel-bin
 bazel-out
 bazel-testlogs
-bazel-wasm-cpp
+bazel-wasm-cpp
+tools/bin/

.licenserc.yaml

@@ -24,6 +24,7 @@ header:
     - 'plugins/**'
     - 'CODEOWNERS'
     - 'VERSION'
+    - 'tools/'
 
   comment: on-failure
 dependency:

Makefile.core.mk

@@ -163,4 +163,47 @@ clean-gateway: clean-istio
 clean-env:
 	rm -rf out/
 
-clean: clean-higress clean-gateway clean-istio clean-env
+clean-tool:
+	rm -rf tools/bin
+
+clean: clean-higress clean-gateway clean-istio clean-env clean-tool
+
+include tools/tools.mk
+
+.PHONY: e2e-test
+e2e-test: $(tools/kind) create-cluster install run-e2e-test delete-cluster
+
+create-cluster: $(tools/kind)
+	tools/hack/create-cluster.sh
+
+.PHONY: delete-cluster
+delete-cluster: $(tools/kind) ## Delete kind cluster.
+	@$(LOG_TARGET)
+	$(tools/kind) delete cluster --name higress
+
+.PHONY: kube-load-image
+kube-load-image: docker-build $(tools/kind) ## Install the EG image to a kind cluster using the provided $IMAGE and $TAG.
+	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG)
+
+.PHONY: run-e2e-test
+run-e2e-test:
+	@echo "Running e2e tests"
+	kubectl apply -f https://github.com/alibaba/higress/releases/download/v0.5.2/quickstart.yaml
+	kubectl wait --timeout=5m -n higress-system deployment/higress-controller --for=condition=Available
+	kubectl wait --timeout=5m -n higress-system deployment/higress-gateway --for=condition=Available
+
+	@curl -i -m 10 http://127.0.0.1:80/foo
+	$(eval STATUS_CODE := $(shell curl -i -m 10 -o /dev/null -w %{http_code} http://127.0.0.1:80/foo))
+	@if [ "${STATUS_CODE}" == "503" ]; then \
+		echo -e "\n${STATUS_CODE}\n"; \
+		exit 1; \
+	fi
+
+	@curl -i -m 10 http://127.0.0.1:80/bar
+	$(eval STATUS_CODE := $(shell curl -i -m 10 -o /dev/null -w %{http_code} http://127.0.0.1:80/bar))
+	@if [ "${STATUS_CODE}" == "503" ]; then \
+		echo -e "\n${STATUS_CODE}\n"; \
+		exit 1; \
+	fi
+
+include tools/lint.mk

tools/hack/cluster.conf

@@ -0,0 +1,32 @@
+# Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#      http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# cluster.conf
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP

tools/hack/create-cluster.sh

@@ -0,0 +1,29 @@
+# Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#      http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Setup default values
+CLUSTER_NAME=${CLUSTER_NAME:-"higress"}
+METALLB_VERSION=${METALLB_VERSION:-"v0.13.7"}
+KIND_NODE_TAG=${KIND_NODE_TAG:-"v1.25.3"}
+
+## Create kind cluster.
+if [[ -z "${KIND_NODE_TAG}" ]]; then
+  tools/bin/kind create cluster --name "${CLUSTER_NAME}" --config=tools/hack/cluster.conf
+else
+  tools/bin/kind create cluster --image "kindest/node:${KIND_NODE_TAG}" --name "${CLUSTER_NAME}" --config=tools/hack/cluster.conf
+fi

tools/hack/kind-load-image.sh

@@ -0,0 +1,56 @@
+# Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#      http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+readonly KIND=${KIND:-tools/bin/kind}
+readonly CLUSTER_NAME=${CLUSTER_NAME:-"higress"}
+
+# Docker variables
+readonly IMAGE="$1"
+readonly TAG="$2"
+
+# Wrap sed to deal with GNU and BSD sed flags.
+run::sed() {
+  if sed --version </dev/null 2>&1 | grep -q GNU; then
+    # GNU sed
+    sed -i "$@"
+  else
+    # assume BSD sed
+    sed -i '' "$@"
+  fi
+}
+
+kind::cluster::exists() {
+    ${KIND} get clusters | grep -q "$1"
+}
+
+kind::cluster::load() {
+    ${KIND} load docker-image \
+        --name "${CLUSTER_NAME}" \
+        "$@"
+}
+
+if ! kind::cluster::exists "$CLUSTER_NAME" ; then
+    echo "cluster $CLUSTER_NAME does not exist"
+    exit 2
+fi
+
+# Push the Higress image to the kind cluster.
+echo "Loading image ${IMAGE}:${TAG} to kind cluster ${CLUSTER_NAME}..."
+kind::cluster::load "${IMAGE}:${TAG}"

tools/lint.mk

@@ -0,0 +1,59 @@
+# This is a wrapper to do lint checks
+#
+# All make targets related to lint are defined in this file.
+
+##@ Lint
+
+GITHUB_ACTION ?=
+
+.PHONY: lint
+lint: ## Run all linter of code sources, including golint, yamllint, whitenoise lint and codespell.
+
+# lint-deps is run separately in CI to separate the tooling install logs from the actual output logs generated
+# by the lint tooling.
+.PHONY: lint-deps
+lint-deps: ## Everything necessary to lint
+
+GOLANGCI_LINT_FLAGS ?= $(if $(GITHUB_ACTION),--out-format=github-actions)
+.PHONY: lint.golint
+lint: lint.golint
+lint-deps: $(tools/golangci-lint)
+lint.golint: $(tools/golangci-lint)
+	$(tools/golangci-lint) run $(GOLANGCI_LINT_FLAGS) --config=tools/linter/golangci-lint/.golangci.yml
+
+.PHONY: lint.yamllint
+lint: lint.yamllint
+lint-deps: $(tools/yamllint)
+lint.yamllint: $(tools/yamllint)
+	$(tools/yamllint) --config-file=tools/linter/yamllint/.yamllint $$(git ls-files :*.yml :*.yaml | xargs -L1 dirname | sort -u) 
+
+CODESPELL_FLAGS ?= $(if $(GITHUB_ACTION),--disable-colors)
+.PHONY: lint.codespell
+lint: lint.codespell
+lint-deps: $(tools/codespell)
+lint.codespell: CODESPELL_SKIP := $(shell cat tools/linter/codespell/.codespell.skip | tr \\n ',')
+lint.codespell: $(tools/codespell)
+
+# This ::add-matcher/::remove-matcher business is based on
+# https://github.com/codespell-project/actions-codespell/blob/2292753ad350451611cafcbabc3abe387491339a/entrypoint.sh
+# We do this here instead of just using
+# codespell-project/codespell-problem-matcher@v1 so that the matcher
+# doesn't apply to the other linters that `make lint` also runs.
+#
+# This recipe is written a little awkwardly with everything running in
+# one shell, this is because we want the ::remove-matcher lines to get
+# printed whether or not it finds complaints.
+	@PS4=; set -e; { \
+	  if test -n "$$GITHUB_ACTION"; then \
+	    printf '::add-matcher::$(CURDIR)/tools/linter/codespell/matcher.json\n'; \
+	    trap "printf '::remove-matcher owner=codespell-matcher-default::\n::remove-matcher owner=codespell-matcher-specified::\n'" EXIT; \
+	  fi; \
+	  (set -x; $(tools/codespell) $(CODESPELL_FLAGS) --skip $(CODESPELL_SKIP) --ignore-words tools/linter/codespell/.codespell.ignorewords --check-filenames --check-hidden -q2); \
+	}
+
+.PHONY: lint.shellcheck
+lint: lint.shellcheck
+lint-deps: $(tools/shellcheck)
+lint.shellcheck: $(tools/shellcheck)
+	$(tools/shellcheck) tools/hack/*.sh
+	$(tools/shellcheck) script/*.sh

tools/linter/codespell/.codespell.ignorewords

@@ -0,0 +1,2 @@
+keypair
+keypairs

tools/linter/codespell/.codespell.skip

@@ -0,0 +1,14 @@
+.git
+.idea
+*.png
+*.woff
+*.woff2
+*.eot
+*.ttf
+*.jpg
+*.ico
+*.svg
+./docs/html/*
+go.mod
+go.sum
+bin

tools/linter/codespell/matcher.json

@@ -0,0 +1,26 @@
+{
+	"problemMatcher": [
+		{
+			"owner": "codespell-matcher-default",
+			"pattern": [
+				{
+					"regexp": "^(.+):(\\d+):\\s+(.+)$",
+					"file": 1,
+					"line": 2,
+					"message": 3
+				}
+			]
+		},
+		{
+			"owner": "codespell-matcher-specified",
+			"pattern": [
+				{
+					"regexp": "^(ERROR|WARNING):\\s+(.+):\\s+(.+?)\\s*$",
+					"file": 3,
+					"severity": 1,
+					"message": 2
+				}
+			]
+		}
+	]
+}

tools/linter/golangci-lint/.golangci.yml

@@ -0,0 +1,46 @@
+run:
+  deadline: 10m
+
+linters:
+  enable:
+    - bodyclose
+    - gofmt
+    - goimports
+    - revive
+    - gosec
+    - misspell
+    - exportloopref
+    - unconvert
+    - unparam
+    - goheader
+    - gocritic
+
+linters-settings:
+  gci:
+    sections:
+      - standard # Captures all standard packages if they do not match another section.
+      - default # Contains all imports that could not be matched to another section type.
+      - prefix(github.com/alibaba/higress/) # Groups all imports with the specified Prefix.
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    local-prefixes: github.com/alibaba/higress/
+  gofmt:
+    simplify: true
+  unparam:
+    check-exported: false
+
+issues:
+  exclude-rules:
+    - path: zz_generated
+      linters:
+        - goimports
+    - linters:
+        - staticcheck
+      text: "SA1019:"
+    - path: test/e2e
+      linters:
+        - bodyclose
+  # Show the complete output
+  max-issues-per-linter: 0
+  max-same-issues: 0