diff --git a/.github/workflows/e2e-test-ci.yml b/.github/workflows/e2e-test-ci.yml index ebe21ab4e0..d6f093ec38 100644 --- a/.github/workflows/e2e-test-ci.yml +++ b/.github/workflows/e2e-test-ci.yml @@ -7,25 +7,16 @@ on: pull_request: branches: - master - - ci/e2e jobs: e2e-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Install minikube + - name: Install kind run: | - sh ./utils/minikube.sh - - name: Output cluster info - run: kubectl cluster-info - - name: Add images - run: | - IMAGE_TAG=dev make build-image-to-minikube - eval $(minikube docker-env) - docker pull apache/apisix:dev - docker pull bitnami/etcd:3.4.14-debian-10-r0 - docker pull kennethreitz/httpbin - docker images + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.10.0/kind-linux-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin - name: Setup Go Env uses: actions/setup-go@v1 with: @@ -37,7 +28,7 @@ jobs: - name: Run e2e test cases working-directory: ./ run: | - make e2e-test E2E_SKIP_BUILD=1 E2E_CONCURRENCY=1 + make e2e-test E2E_CONCURRENCY=2 - name: upload coverage profile working-directory: ./test/e2e run: | diff --git a/Makefile b/Makefile index bb04c5e434..0313bbd902 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ default: help VERSION ?= 0.4.0 RELEASE_SRC = apache-apisix-ingress-controller-${VERSION}-src +LOCAL_REGISTRY="localhost:5000" IMAGE_TAG ?= dev GINKGO ?= $(shell which ginkgo) @@ -36,31 +37,37 @@ GO_LDFLAGS ?= "-X=$(VERSYM)=$(VERSION) -X=$(GITSHASYM)=$(GITSHA) -X=$(BUILDOSSYM E2E_CONCURRENCY ?= 1 E2E_SKIP_BUILD ?= 0 -### build: Build apisix-ingress-controller +### build: Build apisix-ingress-controller +.PHONY: build build: go build \ -o apisix-ingress-controller \ -ldflags $(GO_LDFLAGS) \ main.go -### build-image: Build apisix-ingress-controller image +### build-image: Build apisix-ingress-controller image +.PHONY: build-image build-image: docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) . -### lint: Do static lint check +### lint: Do static lint check +.PHONY: lint lint: golangci-lint run -### gofmt: Format all go codes +### gofmt: Format all go codes +.PHONY: gofmt gofmt: find . -type f -name "*.go" | xargs gofmt -w -s -### unit-test: Run unit test cases +### unit-test: Run unit test cases +.PHONY: unit-test unit-test: go test -cover -coverprofile=coverage.txt ./... -### e2e-test: Run e2e test cases (minikube is required) -e2e-test: ginkgo-check build-image-to-minikube +### e2e-test: Run e2e test cases (kind is required) +.PHONY: e2e-test +e2e-test: ginkgo-check push-images-to-kind kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixRoute.yaml kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixUpstream.yaml kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixTls.yaml @@ -73,16 +80,38 @@ ifeq ("$(wildcard $(GINKGO))", "") exit 1 endif -# build images to minikube node directly, it's an internal directive, so don't -# expose it's help message. -build-image-to-minikube: +### push-images-to-kind: Push images used in e2e test suites to kind. +.PHONY: push-images-to-kind +push-images-to-kind: kind-up ifeq ($(E2E_SKIP_BUILD), 0) - @minikube version > /dev/null 2>&1 || (echo "ERROR: minikube is required."; exit 1) - @eval $$(minikube docker-env);\ + docker pull apache/apisix:dev + docker tag apache/apisix:dev $(LOCAL_REGISTRY)/apache/apisix:dev + docker push $(LOCAL_REGISTRY)/apache/apisix:dev + + docker pull bitnami/etcd:3.4.14-debian-10-r0 + docker tag bitnami/etcd:3.4.14-debian-10-r0 $(LOCAL_REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0 + docker push $(LOCAL_REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0 + + docker pull kennethreitz/httpbin + docker tag kennethreitz/httpbin $(LOCAL_REGISTRY)/kennethreitz/httpbin + docker push $(LOCAL_REGISTRY)/kennethreitz/httpbin + docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) . + docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(LOCAL_REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG) + docker push $(LOCAL_REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG) endif -### license-check: Do Apache License Header check +### kind-up: Launch a Kubernetes cluster with a image registry by Kind. +.PHONY: kind-up +kind-up: + ./utils/kind-with-registry.sh +### kind-reset: Delete the Kubernetes cluster created by "make kind-up" +.PHONY: kind-reset +kind-reset: + kind delete cluster --name apisix + +### license-check: Do Apache License Header check +.PHONY: license-check license-check: ifeq ("$(wildcard .actions/openwhisk-utilities/scancode/scanCode.py)", "") git clone https://github.com/apache/openwhisk-utilities.git .actions/openwhisk-utilities @@ -90,13 +119,15 @@ ifeq ("$(wildcard .actions/openwhisk-utilities/scancode/scanCode.py)", "") endif .actions/openwhisk-utilities/scancode/scanCode.py --config .actions/ASF-Release.cfg ./ -### help: Show Makefile rules +### help: Show Makefile rules +.PHONY: help help: @echo Makefile rules: @echo @grep -E '^### [-A-Za-z0-9_]+:' Makefile | sed 's/###/ /' -### release-src: Release source +### release-src: Release source +.PHONY: release-src release-src: tar -zcvf $(RELEASE_SRC).tgz \ --exclude .github \ @@ -118,6 +149,3 @@ release-src: mv $(RELEASE_SRC).tgz release/$(RELEASE_SRC).tgz mv $(RELEASE_SRC).tgz.asc release/$(RELEASE_SRC).tgz.asc mv $(RELEASE_SRC).tgz.sha512 release/$(RELEASE_SRC).tgz.sha512 - -.PHONY: build lint help - diff --git a/test/e2e/README.md b/test/e2e/README.md index bc35f93050..c986b7a079 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -20,8 +20,6 @@ apisix ingress controller e2e test suites ========================================= -For running e2e test cases, a Kubernetes cluster is required, [minikube](https://minikube.sigs.k8s.io/docs/start/) is a good choice to build k8s cluster in development environment. - Scaffold --------- @@ -44,3 +42,17 @@ Features -------- Test caes inside `features` directory test some features about APISIX, such as traffic-split, health check and so on. + +Quick Start +----------- + +Run `make e2e-test` to run the e2e test suites in your development environment, a several stuffs that this command will do: + +1. Create a Kubernetes cluster by [kind](https://kind.sigs.k8s.io/), please installing in advance. +2. Build and push all related images to this cluster. +3. Run e2e test suites. + +Step `1` and `2` can be skipped by passing `E2E_SKIP_BUILD=1` to this directive, also, you can customize the +running concurrency of e2e test suites by passing `E2E_CONCURRENCY=X` where `X` is the desired number of cases running in parallel. + +Run `make kind-reset` to delete the cluster that created by `make e2e-test`. diff --git a/test/e2e/scaffold/apisix.go b/test/e2e/scaffold/apisix.go index f4448feba0..5719f45d4d 100644 --- a/test/e2e/scaffold/apisix.go +++ b/test/e2e/scaffold/apisix.go @@ -15,10 +15,7 @@ package scaffold import ( - "errors" "fmt" - "net" - "strconv" "strings" "github.com/gruntwork-io/terratest/modules/k8s" @@ -77,7 +74,7 @@ spec: tcpSocket: port: 9080 timeoutSeconds: 2 - image: "apache/apisix:dev" + image: "localhost:5000/apache/apisix:dev" imagePullPolicy: IfNotPresent name: apisix-deployment-e2e-test ports: @@ -127,63 +124,6 @@ spec: ` ) -func (s *Scaffold) apisixServiceURL() (string, error) { - if len(s.nodes) == 0 { - return "", errors.New("no available node") - } - var addr string - for _, node := range s.nodes { - if len(node.Status.Addresses) > 0 { - addr = node.Status.Addresses[0].Address - break - } - } - for _, port := range s.apisixService.Spec.Ports { - if port.Name == "http" { - return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil - } - } - return "", errors.New("no http port in apisix service") -} - -func (s *Scaffold) apisixServiceHttpsURL() (string, error) { - if len(s.nodes) == 0 { - return "", errors.New("no available node") - } - var addr string - for _, node := range s.nodes { - if len(node.Status.Addresses) > 0 { - addr = node.Status.Addresses[0].Address - break - } - } - for _, port := range s.apisixService.Spec.Ports { - if port.Name == "https" { - return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil - } - } - return "", errors.New("no https port defined in apisix service") -} - -func (s *Scaffold) apisixAdminServiceURL() (string, error) { - if len(s.nodes) == 0 { - return "", errors.New("no available node") - } - var addr string - for _, node := range s.nodes { - if len(node.Status.Addresses) > 0 { - addr = node.Status.Addresses[0].Address - break - } - } - for _, port := range s.apisixService.Spec.Ports { - if port.Name == "http-admin" { - return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil - } - } - return "", errors.New("no http-admin port in apisix admin service") -} - func (s *Scaffold) newAPISIX() (*corev1.Service, error) { defaultData, err := s.renderConfig(s.opts.APISIXDefaultConfigPath) if err != nil { @@ -210,6 +150,7 @@ func (s *Scaffold) newAPISIX() (*corev1.Service, error) { if err != nil { return nil, err } + return svc, nil } diff --git a/test/e2e/scaffold/etcd.go b/test/e2e/scaffold/etcd.go index de227e1583..c0559bb69a 100644 --- a/test/e2e/scaffold/etcd.go +++ b/test/e2e/scaffold/etcd.go @@ -65,7 +65,7 @@ spec: tcpSocket: port: 2379 timeoutSeconds: 2 - image: "bitnami/etcd:3.4.14-debian-10-r0" + image: "localhost:5000/bitnami/etcd:3.4.14-debian-10-r0" imagePullPolicy: IfNotPresent name: etcd-deployment-e2e-test ports: diff --git a/test/e2e/scaffold/httpbin.go b/test/e2e/scaffold/httpbin.go index 41701d6ea2..d9fd861942 100644 --- a/test/e2e/scaffold/httpbin.go +++ b/test/e2e/scaffold/httpbin.go @@ -64,7 +64,7 @@ spec: tcpSocket: port: 80 timeoutSeconds: 2 - image: "kennethreitz/httpbin" + image: "localhost:5000/kennethreitz/httpbin" imagePullPolicy: IfNotPresent name: httpbin-deployment-e2e-test ports: diff --git a/test/e2e/scaffold/ingress.go b/test/e2e/scaffold/ingress.go index b3cb70fdb9..67f9343038 100644 --- a/test/e2e/scaffold/ingress.go +++ b/test/e2e/scaffold/ingress.go @@ -227,8 +227,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: "apache/apisix-ingress-controller:dev" - imagePullPolicy: Never + image: "localhost:5000/apache/apisix-ingress-controller:dev" + imagePullPolicy: Always name: ingress-apisix-controller-deployment-e2e-test ports: - containerPort: 8080 @@ -268,11 +268,11 @@ func (s *Scaffold) newIngressAPISIXController() error { if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, crb); err != nil { return err } - s.addFinializer(func() { + s.addFinalizers(func() { err := k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, crb) assert.Nil(s.t, err, "deleting ClusterRoleBinding") }) - s.addFinializer(func() { + s.addFinalizers(func() { err := k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, cr) assert.Nil(s.t, err, "deleting ClusterRole") }) diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go index 0c3fec9c4c..a87a55f36c 100644 --- a/test/e2e/scaffold/k8s.go +++ b/test/e2e/scaffold/k8s.go @@ -109,7 +109,7 @@ func (s *Scaffold) CreateResourceFromStringWithNamespace(yaml, namespace string) defer func() { s.kubectlOptions.Namespace = originalNamespace }() - s.addFinializer(func() { + s.addFinalizers(func() { originalNamespace := s.kubectlOptions.Namespace s.kubectlOptions.Namespace = namespace defer func() { @@ -154,13 +154,9 @@ func ensureNumApisixCRDsCreated(url string, desired int) error { // EnsureNumApisixRoutesCreated waits until desired number of Routes are created in // APISIX cluster. func (s *Scaffold) EnsureNumApisixRoutesCreated(desired int) error { - host, err := s.apisixAdminServiceURL() - if err != nil { - return err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin/routes", } return ensureNumApisixCRDsCreated(u.String(), desired) @@ -169,13 +165,9 @@ func (s *Scaffold) EnsureNumApisixRoutesCreated(desired int) error { // EnsureNumApisixUpstreamsCreated waits until desired number of Upstreams are created in // APISIX cluster. func (s *Scaffold) EnsureNumApisixUpstreamsCreated(desired int) error { - host, err := s.apisixAdminServiceURL() - if err != nil { - return err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin/upstreams", } return ensureNumApisixCRDsCreated(u.String(), desired) @@ -183,13 +175,9 @@ func (s *Scaffold) EnsureNumApisixUpstreamsCreated(desired int) error { // ListApisixUpstreams list all upstreams from APISIX func (s *Scaffold) ListApisixUpstreams() ([]*v1.Upstream, error) { - host, err := s.apisixAdminServiceURL() - if err != nil { - return nil, err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin", } cli, err := apisix.NewClient() @@ -207,13 +195,9 @@ func (s *Scaffold) ListApisixUpstreams() ([]*v1.Upstream, error) { // ListApisixRoutes list all routes from APISIX. func (s *Scaffold) ListApisixRoutes() ([]*v1.Route, error) { - host, err := s.apisixAdminServiceURL() - if err != nil { - return nil, err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin", } cli, err := apisix.NewClient() @@ -231,13 +215,9 @@ func (s *Scaffold) ListApisixRoutes() ([]*v1.Route, error) { // ListApisixTls list all ssl from APISIX func (s *Scaffold) ListApisixTls() ([]*v1.Ssl, error) { - host, err := s.apisixAdminServiceURL() - if err != nil { - return nil, err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin", } cli, err := apisix.NewClient() @@ -252,3 +232,47 @@ func (s *Scaffold) ListApisixTls() ([]*v1.Ssl, error) { } return cli.Cluster("").SSL().List(context.TODO()) } + +func (s *Scaffold) newAPISIXTunnels() error { + var ( + adminNodePort int + httpNodePort int + httpsNodePort int + adminPort int + httpPort int + httpsPort int + ) + for _, port := range s.apisixService.Spec.Ports { + if port.Name == "http" { + httpNodePort = int(port.NodePort) + httpPort = int(port.Port) + } else if port.Name == "https" { + httpsNodePort = int(port.NodePort) + httpsPort = int(port.Port) + } else if port.Name == "http-admin" { + adminNodePort = int(port.NodePort) + adminPort = int(port.Port) + } + } + + s.apisixAdminTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test", + adminNodePort, adminPort) + s.apisixHttpTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test", + httpNodePort, httpPort) + s.apisixHttpsTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test", + httpsNodePort, httpsPort) + + if err := s.apisixAdminTunnel.ForwardPortE(s.t); err != nil { + return err + } + s.addFinalizers(s.apisixAdminTunnel.Close) + if err := s.apisixHttpTunnel.ForwardPortE(s.t); err != nil { + return err + } + s.addFinalizers(s.apisixHttpTunnel.Close) + if err := s.apisixHttpsTunnel.ForwardPortE(s.t); err != nil { + return err + } + s.addFinalizers(s.apisixHttpsTunnel.Close) + return nil +} diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 7a9da7e6ef..d64e07d28e 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -62,6 +62,10 @@ type Scaffold struct { httpbinService *corev1.Service finializers []func() + apisixAdminTunnel *k8s.Tunnel + apisixHttpTunnel *k8s.Tunnel + apisixHttpsTunnel *k8s.Tunnel + // Used for template rendering. EtcdServiceFQDN string } @@ -137,18 +141,11 @@ func (s *Scaffold) DefaultHTTPBackend() (string, []int32) { return s.httpbinService.Name, ports } -// GetAPISIXEndpoint returns the service and port (as an endpoint). -func (s *Scaffold) GetAPISIXEndpoint() (string, error) { - return s.apisixServiceURL() -} - // NewAPISIXClient creates the default HTTP client. func (s *Scaffold) NewAPISIXClient() *httpexpect.Expect { - host, err := s.apisixServiceURL() - assert.Nil(s.t, err, "getting apisix service url") u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixHttpTunnel.Endpoint(), } return httpexpect.WithConfig(httpexpect.Config{ BaseURL: u.String(), @@ -166,11 +163,9 @@ func (s *Scaffold) NewAPISIXClient() *httpexpect.Expect { // NewAPISIXHttpsClient creates the default HTTPs client. func (s *Scaffold) NewAPISIXHttpsClient() *httpexpect.Expect { - host, err := s.apisixServiceHttpsURL() - assert.Nil(s.t, err, "getting apisix service url") u := url.URL{ Scheme: "https", - Host: host, + Host: s.apisixHttpsTunnel.Endpoint(), } return httpexpect.WithConfig(httpexpect.Config{ BaseURL: u.String(), @@ -213,6 +208,9 @@ func (s *Scaffold) beforeEach() { err = s.waitAllAPISIXPodsAvailable() assert.Nil(s.t, err, "waiting for apisix ready") + err = s.newAPISIXTunnels() + assert.Nil(s.t, err, "creating apisix tunnels") + s.httpbinService, err = s.newHTTPBIN() assert.Nil(s.t, err, "initializing httpbin") @@ -239,7 +237,7 @@ func (s *Scaffold) afterEach() { time.Sleep(3 * time.Second) } -func (s *Scaffold) addFinializer(f func()) { +func (s *Scaffold) addFinalizers(f func()) { s.finializers = append(s.finializers, f) } diff --git a/utils/kind-with-registry.sh b/utils/kind-with-registry.sh new file mode 100755 index 0000000000..ea01a2c6bb --- /dev/null +++ b/utils/kind-with-registry.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -o errexit +set -o nounset +set -o pipefail + +# desired cluster name; default is "apisix" +KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-apisix}" + +if kind get clusters | grep -q ^apisix$ ; then + echo "cluster already exists, moving on" + exit 0 +fi + +# create registry container unless it already exists +kind_version=$(kind version) +kind_network='kind' +reg_name='kind-registry' +reg_port='5000' +case "${kind_version}" in + "kind v0.7."* | "kind v0.6."* | "kind v0.5."*) + kind_network='bridge' + ;; +esac + +# create registry container unless it already exists +running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" +if [ "${running}" != 'true' ]; then + docker run \ + -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +reg_host="${reg_name}" +if [ "${kind_network}" = "bridge" ]; then + reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" +fi +echo "Registry Host: ${reg_host}" + +# create a cluster with the local registry enabled in containerd +cat <