chore: use kind to run e2e test suites

.github/workflows/e2e-test-ci.yml

@@ -7,25 +7,16 @@ on:
   pull_request:
     branches:
       - master
-      - ci/e2e
 jobs:
   e2e-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install minikube
+      - name: Install kind
         run: |
-          sh ./utils/minikube.sh
-      - name: Output cluster info
-        run: kubectl cluster-info
-      - name: Add images
-        run: |
-          IMAGE_TAG=dev make build-image-to-minikube
-          eval $(minikube docker-env)
-          docker pull apache/apisix:dev
-          docker pull bitnami/etcd:3.4.14-debian-10-r0
-          docker pull kennethreitz/httpbin
-          docker images
+          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.10.0/kind-linux-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
       - name: Setup Go Env
         uses: actions/setup-go@v1
         with:
@@ -37,7 +28,7 @@ jobs:
       - name: Run e2e test cases
         working-directory: ./
         run: |
-          make e2e-test E2E_SKIP_BUILD=1 E2E_CONCURRENCY=1
+          make e2e-test E2E_CONCURRENCY=2
       - name: upload coverage profile
         working-directory: ./test/e2e
         run: |

Makefile

@@ -18,6 +18,7 @@ default: help
 
 VERSION ?= 0.4.0
 RELEASE_SRC = apache-apisix-ingress-controller-${VERSION}-src
+LOCAL_REGISTRY="localhost:5000"
 IMAGE_TAG ?= dev
 
 GINKGO ?= $(shell which ginkgo)
@@ -36,31 +37,37 @@ GO_LDFLAGS ?= "-X=$(VERSYM)=$(VERSION) -X=$(GITSHASYM)=$(GITSHA) -X=$(BUILDOSSYM
 E2E_CONCURRENCY ?= 1
 E2E_SKIP_BUILD ?= 0
 
-### build:            Build apisix-ingress-controller
+### build:                Build apisix-ingress-controller
+.PHONY: build
 build:
 	go build \
 		-o apisix-ingress-controller \
 		-ldflags $(GO_LDFLAGS) \
 		main.go
 
-### build-image:      Build apisix-ingress-controller image
+### build-image:          Build apisix-ingress-controller image
+.PHONY: build-image
 build-image:
 	docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) .
 
-### lint:             Do static lint check
+### lint:                 Do static lint check
+.PHONY: lint
 lint:
 	golangci-lint run
 
-### gofmt:            Format all go codes
+### gofmt:                Format all go codes
+.PHONY: gofmt
 gofmt:
 	find . -type f -name "*.go" | xargs gofmt -w -s
 
-### unit-test:        Run unit test cases
+### unit-test:            Run unit test cases
+.PHONY: unit-test
 unit-test:
 	go test -cover -coverprofile=coverage.txt ./...
 
-### e2e-test:         Run e2e test cases (minikube is required)
-e2e-test: ginkgo-check build-image-to-minikube
+### e2e-test:             Run e2e test cases (kind is required)
+.PHONY: e2e-test
+e2e-test: ginkgo-check push-images-to-kind
 	kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixRoute.yaml
 	kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixUpstream.yaml
 	kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixTls.yaml
@@ -73,30 +80,54 @@ ifeq ("$(wildcard $(GINKGO))", "")
 	exit 1
 endif
 
-# build images to minikube node directly, it's an internal directive, so don't
-# expose it's help message.
-build-image-to-minikube:
+### push-images-to-kind:  Push images used in e2e test suites to kind.
+.PHONY: push-images-to-kind
+push-images-to-kind: kind-up
 ifeq ($(E2E_SKIP_BUILD), 0)
-	@minikube version > /dev/null 2>&1 || (echo "ERROR: minikube is required."; exit 1)
-	@eval $$(minikube docker-env);\
+	docker pull apache/apisix:dev
+	docker tag apache/apisix:dev $(LOCAL_REGISTRY)/apache/apisix:dev
+	docker push $(LOCAL_REGISTRY)/apache/apisix:dev
+
+	docker pull bitnami/etcd:3.4.14-debian-10-r0
+	docker tag bitnami/etcd:3.4.14-debian-10-r0 $(LOCAL_REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0
+	docker push $(LOCAL_REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0
+
+	docker pull kennethreitz/httpbin
+	docker tag kennethreitz/httpbin $(LOCAL_REGISTRY)/kennethreitz/httpbin
+	docker push $(LOCAL_REGISTRY)/kennethreitz/httpbin
+
 	docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) .
+	docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(LOCAL_REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
+	docker push $(LOCAL_REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
 endif
 
-### license-check:    Do Apache License Header check
+### kind-up:              Launch a Kubernetes cluster with a image registry by Kind.
+.PHONY: kind-up
+kind-up:
+	./utils/kind-with-registry.sh
+### kind-reset:           Delete the Kubernetes cluster created by "make kind-up"
+.PHONY: kind-reset
+kind-reset:
+	kind delete cluster --name apisix
+
+### license-check:        Do Apache License Header check
+.PHONY: license-check
 license-check:
 ifeq ("$(wildcard .actions/openwhisk-utilities/scancode/scanCode.py)", "")
 	git clone https://github.com/apache/openwhisk-utilities.git .actions/openwhisk-utilities
 	cp .actions/ASF* .actions/openwhisk-utilities/scancode/
 endif
 	.actions/openwhisk-utilities/scancode/scanCode.py --config .actions/ASF-Release.cfg ./
 
-### help:             Show Makefile rules
+### help:                 Show Makefile rules
+.PHONY: help
 help:
 	@echo Makefile rules:
 	@echo
 	@grep -E '^### [-A-Za-z0-9_]+:' Makefile | sed 's/###/   /'
 
-### release-src:      Release source
+### release-src:          Release source
+.PHONY: release-src
 release-src:
 	tar -zcvf $(RELEASE_SRC).tgz \
 	--exclude .github \
@@ -118,6 +149,3 @@ release-src:
 	mv $(RELEASE_SRC).tgz release/$(RELEASE_SRC).tgz
 	mv $(RELEASE_SRC).tgz.asc release/$(RELEASE_SRC).tgz.asc
 	mv $(RELEASE_SRC).tgz.sha512 release/$(RELEASE_SRC).tgz.sha512
-
-.PHONY: build lint help
-

test/e2e/README.md

@@ -20,8 +20,6 @@
 apisix ingress controller e2e test suites
 =========================================
 
-For running e2e test cases, a Kubernetes cluster is required, [minikube](https://minikube.sigs.k8s.io/docs/start/) is a good choice to build k8s cluster in development environment.
-
 Scaffold
 ---------
 
@@ -44,3 +42,17 @@ Features
 --------
 
 Test caes inside `features` directory test some features about APISIX, such as traffic-split, health check and so on.
+
+Quick Start
+-----------
+
+Run `make e2e-test` to run the e2e test suites in your development environment, a several stuffs that this command will do:
+
+1. Create a Kubernetes cluster by [kind](https://kind.sigs.k8s.io/), please installing in advance.
+2. Build and push all related images to this cluster.
+3. Run e2e test suites.
+
+Step `1` and `2` can be skipped by passing `E2E_SKIP_BUILD=1` to this directive, also, you can customize the
+running concurrency of e2e test suites by passing `E2E_CONCURRENCY=X` where `X` is the desired number of cases running in parallel.
+
+Run `make kind-reset` to delete the cluster that created by `make e2e-test`.

test/e2e/scaffold/apisix.go

@@ -15,10 +15,7 @@
 package scaffold
 
 import (
-	"errors"
 	"fmt"
-	"net"
-	"strconv"
 	"strings"
 
 	"github.com/gruntwork-io/terratest/modules/k8s"
@@ -77,7 +74,7 @@ spec:
             tcpSocket:
               port: 9080
             timeoutSeconds: 2
-          image: "apache/apisix:dev"
+          image: "localhost:5000/apache/apisix:dev"
           imagePullPolicy: IfNotPresent
           name: apisix-deployment-e2e-test
           ports:
@@ -127,63 +124,6 @@ spec:
 `
 )
 
-func (s *Scaffold) apisixServiceURL() (string, error) {
-	if len(s.nodes) == 0 {
-		return "", errors.New("no available node")
-	}
-	var addr string
-	for _, node := range s.nodes {
-		if len(node.Status.Addresses) > 0 {
-			addr = node.Status.Addresses[0].Address
-			break
-		}
-	}
-	for _, port := range s.apisixService.Spec.Ports {
-		if port.Name == "http" {
-			return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil
-		}
-	}
-	return "", errors.New("no http port in apisix service")
-}
-
-func (s *Scaffold) apisixServiceHttpsURL() (string, error) {
-	if len(s.nodes) == 0 {
-		return "", errors.New("no available node")
-	}
-	var addr string
-	for _, node := range s.nodes {
-		if len(node.Status.Addresses) > 0 {
-			addr = node.Status.Addresses[0].Address
-			break
-		}
-	}
-	for _, port := range s.apisixService.Spec.Ports {
-		if port.Name == "https" {
-			return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil
-		}
-	}
-	return "", errors.New("no https port defined in apisix service")
-}
-
-func (s *Scaffold) apisixAdminServiceURL() (string, error) {
-	if len(s.nodes) == 0 {
-		return "", errors.New("no available node")
-	}
-	var addr string
-	for _, node := range s.nodes {
-		if len(node.Status.Addresses) > 0 {
-			addr = node.Status.Addresses[0].Address
-			break
-		}
-	}
-	for _, port := range s.apisixService.Spec.Ports {
-		if port.Name == "http-admin" {
-			return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil
-		}
-	}
-	return "", errors.New("no http-admin port in apisix admin service")
-}
-
 func (s *Scaffold) newAPISIX() (*corev1.Service, error) {
 	defaultData, err := s.renderConfig(s.opts.APISIXDefaultConfigPath)
 	if err != nil {
@@ -210,6 +150,7 @@ func (s *Scaffold) newAPISIX() (*corev1.Service, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	return svc, nil
 }
 

test/e2e/scaffold/etcd.go

@@ -65,7 +65,7 @@ spec:
             tcpSocket:
               port: 2379
             timeoutSeconds: 2
-          image: "bitnami/etcd:3.4.14-debian-10-r0"
+          image: "localhost:5000/bitnami/etcd:3.4.14-debian-10-r0"
           imagePullPolicy: IfNotPresent
           name: etcd-deployment-e2e-test
           ports:

test/e2e/scaffold/httpbin.go

@@ -64,7 +64,7 @@ spec:
             tcpSocket:
               port: 80
             timeoutSeconds: 2
-          image: "kennethreitz/httpbin"
+          image: "localhost:5000/kennethreitz/httpbin"
           imagePullPolicy: IfNotPresent
           name: httpbin-deployment-e2e-test
           ports:

test/e2e/scaffold/ingress.go

@@ -227,8 +227,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-          image: "apache/apisix-ingress-controller:dev"
-          imagePullPolicy: Never
+          image: "localhost:5000/apache/apisix-ingress-controller:dev"
+          imagePullPolicy: Always
           name: ingress-apisix-controller-deployment-e2e-test
           ports:
             - containerPort: 8080
@@ -268,11 +268,11 @@ func (s *Scaffold) newIngressAPISIXController() error {
 	if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, crb); err != nil {
 		return err
 	}
-	s.addFinializer(func() {
+	s.addFinalizers(func() {
 		err := k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, crb)
 		assert.Nil(s.t, err, "deleting ClusterRoleBinding")
 	})
-	s.addFinializer(func() {
+	s.addFinalizers(func() {
 		err := k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, cr)
 		assert.Nil(s.t, err, "deleting ClusterRole")
 	})