From 42991d2cae27e2a7722cb26721724ed40b41059a Mon Sep 17 00:00:00 2001 From: Alex Zhang Date: Thu, 1 Apr 2021 21:50:48 +0800 Subject: [PATCH 1/5] test: add e2e test cases for request-id (#327) --- test/e2e/plugins/request_id.go | 110 +++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 test/e2e/plugins/request_id.go diff --git a/test/e2e/plugins/request_id.go b/test/e2e/plugins/request_id.go new file mode 100644 index 0000000000..3943588c93 --- /dev/null +++ b/test/e2e/plugins/request_id.go @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package plugins + +import ( + "fmt" + + "github.com/stretchr/testify/assert" + + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" + "github.com/onsi/ginkgo" +) + +var _ = ginkgo.Describe("request-id plugin", func() { + opts := &scaffold.Options{ + Name: "default", + Kubeconfig: scaffold.GetKubeconfig(), + APISIXConfigPath: "testdata/apisix-gw-config.yaml", + APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml", + IngressAPISIXReplicas: 1, + HTTPBinServicePort: 80, + APISIXRouteVersion: "apisix.apache.org/v2alpha1", + } + s := scaffold.NewScaffold(opts) + ginkgo.It("sanity", func() { + backendSvc, backendPorts := s.DefaultHTTPBackend() + ar := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2alpha1 +kind: ApisixRoute +metadata: + name: httpbin-route +spec: + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /ip + backends: + - serviceName: %s + servicePort: %d + weight: 10 + plugins: + - name: request-id + enable: true +`, backendSvc, backendPorts[0]) + + assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ar)) + + err := s.EnsureNumApisixUpstreamsCreated(1) + assert.Nil(ginkgo.GinkgoT(), err, "Checking number of upstreams") + err = s.EnsureNumApisixRoutesCreated(1) + assert.Nil(ginkgo.GinkgoT(), err, "Checking number of routes") + + resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").Expect() + resp.Status(200) + resp.Header("X-Request-Id").NotEmpty() + resp.Body().Contains("origin") + }) + + ginkgo.It("disable plugin", func() { + backendSvc, backendPorts := s.DefaultHTTPBackend() + ar := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2alpha1 +kind: ApisixRoute +metadata: + name: httpbin-route +spec: + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /ip + backends: + - serviceName: %s + servicePort: %d + weight: 10 + plugins: + - name: request-id + enable: false +`, backendSvc, backendPorts[0]) + + assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ar)) + + err := s.EnsureNumApisixUpstreamsCreated(1) + assert.Nil(ginkgo.GinkgoT(), err, "Checking number of upstreams") + err = s.EnsureNumApisixRoutesCreated(1) + assert.Nil(ginkgo.GinkgoT(), err, "Checking number of routes") + + resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org").Expect() + resp.Status(200) + resp.Header("X-Request-Id").Empty() + resp.Body().Contains("origin") + }) +}) From 0d488ce73264c24a68887ea0b9e1d8d45be7e801 Mon Sep 17 00:00:00 2001 From: Alex Zhang Date: Thu, 1 Apr 2021 21:51:50 +0800 Subject: [PATCH 2/5] test: add e2e test cases for limit-count plugin (#328) * test: add e2e test cases for limit-count plugin * fix --- test/e2e/plugins/limit_count.go | 147 ++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 test/e2e/plugins/limit_count.go diff --git a/test/e2e/plugins/limit_count.go b/test/e2e/plugins/limit_count.go new file mode 100644 index 0000000000..e16b615431 --- /dev/null +++ b/test/e2e/plugins/limit_count.go @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package plugins + +import ( + "fmt" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" + "github.com/onsi/ginkgo" +) + +var _ = ginkgo.Describe("limit-count plugin", func() { + opts := &scaffold.Options{ + Name: "default", + Kubeconfig: scaffold.GetKubeconfig(), + APISIXConfigPath: "testdata/apisix-gw-config.yaml", + APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml", + IngressAPISIXReplicas: 1, + HTTPBinServicePort: 80, + APISIXRouteVersion: "apisix.apache.org/v2alpha1", + } + s := scaffold.NewScaffold(opts) + ginkgo.It("localized dimension, limited by remote address", func() { + backendSvc, backendPorts := s.DefaultHTTPBackend() + ar := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2alpha1 +kind: ApisixRoute +metadata: + name: httpbin-route +spec: + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /ip + backends: + - serviceName: %s + servicePort: %d + weight: 10 + plugins: + - name: limit-count + enable: true + config: + rejected_code: 503 + count: 2 + time_window: 3 + key: remote_addr +`, backendSvc, backendPorts[0]) + + assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ar)) + + err := s.EnsureNumApisixUpstreamsCreated(1) + assert.Nil(ginkgo.GinkgoT(), err, "Checking number of upstreams") + err = s.EnsureNumApisixRoutesCreated(1) + assert.Nil(ginkgo.GinkgoT(), err, "Checking number of routes") + + s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org"). + Expect(). + Status(200). + Body(). + Contains("origin") + s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org"). + Expect(). + Status(200). + Body(). + Contains("origin") + s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org"). + Expect(). + Status(503) + time.Sleep(3 * time.Second) + s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org"). + Expect(). + Status(200). + Body(). + Contains("origin") + }) + + ginkgo.It("disable plugin", func() { + backendSvc, backendPorts := s.DefaultHTTPBackend() + ar := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2alpha1 +kind: ApisixRoute +metadata: + name: httpbin-route +spec: + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /ip + backends: + - serviceName: %s + servicePort: %d + weight: 10 + plugins: + - name: limit-count + enable: false + config: + rejected_code: 503 + count: 2 + time_window: 3 + key: remote_addr +`, backendSvc, backendPorts[0]) + + assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ar)) + + err := s.EnsureNumApisixUpstreamsCreated(1) + assert.Nil(ginkgo.GinkgoT(), err, "Checking number of upstreams") + err = s.EnsureNumApisixRoutesCreated(1) + assert.Nil(ginkgo.GinkgoT(), err, "Checking number of routes") + + s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org"). + Expect(). + Status(200). + Body(). + Contains("origin") + s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org"). + Expect(). + Status(200). + Body(). + Contains("origin") + s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.org"). + Expect(). + Status(200). + Body(). + Contains("origin") + }) +}) From 327f842ba09d809d609e2073fa8e8d3c35d49acb Mon Sep 17 00:00:00 2001 From: Alex Zhang Date: Thu, 1 Apr 2021 21:53:46 +0800 Subject: [PATCH 3/5] fix: priority in route rules is not passed to APISIX (#329) --- pkg/apisix/resource.go | 2 + pkg/apisix/route.go | 3 ++ pkg/kube/translation/apisix_route.go | 1 + test/e2e/ingress/resourcepushing.go | 57 ++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) diff --git a/pkg/apisix/resource.go b/pkg/apisix/resource.go index 7d18494f3a..7dadb3e6d5 100644 --- a/pkg/apisix/resource.go +++ b/pkg/apisix/resource.go @@ -81,6 +81,7 @@ type routeItem struct { Uris []string `json:"uris"` Desc string `json:"desc"` Methods []string `json:"methods"` + Priority int `json:"priority"` Plugins map[string]interface{} `json:"plugins"` } @@ -114,6 +115,7 @@ func (i *item) route(clusterName string) (*v1.Route, error) { UpstreamId: route.UpstreamId, ServiceId: route.ServiceId, Plugins: route.Plugins, + Priority: route.Priority, }, nil } diff --git a/pkg/apisix/route.go b/pkg/apisix/route.go index 2da945dcce..5d3ab207a2 100644 --- a/pkg/apisix/route.go +++ b/pkg/apisix/route.go @@ -31,6 +31,7 @@ import ( type routeReqBody struct { Desc string `json:"desc,omitempty"` URI string `json:"uri,omitempty"` + Priority int `json:"priority,omitempty"` Uris []string `json:"uris,omitempty"` Vars [][]v1.StringOrSlice `json:"vars,omitempty"` Host string `json:"host,omitempty"` @@ -159,6 +160,7 @@ func (r *routeClient) Create(ctx context.Context, obj *v1.Route) (*v1.Route, err return nil, err } data, err := json.Marshal(routeReqBody{ + Priority: obj.Priority, Desc: obj.Name, URI: obj.Path, Host: obj.Host, @@ -227,6 +229,7 @@ func (r *routeClient) Update(ctx context.Context, obj *v1.Route) (*v1.Route, err return nil, err } body, err := json.Marshal(routeReqBody{ + Priority: obj.Priority, Desc: obj.Name, Host: obj.Host, URI: obj.Path, diff --git a/pkg/kube/translation/apisix_route.go b/pkg/kube/translation/apisix_route.go index 770a04089e..52c8364b87 100644 --- a/pkg/kube/translation/apisix_route.go +++ b/pkg/kube/translation/apisix_route.go @@ -168,6 +168,7 @@ func (t *translator) TranslateRouteV2alpha1(ar *configv2alpha1.ApisixRoute) ([]* ID: id.GenID(routeName), ResourceVersion: ar.ResourceVersion, }, + Priority: part.Priority, Vars: exprs, Hosts: part.Match.Hosts, Uris: part.Match.Paths, diff --git a/test/e2e/ingress/resourcepushing.go b/test/e2e/ingress/resourcepushing.go index 0e2db4e0b5..6e3bbe609a 100644 --- a/test/e2e/ingress/resourcepushing.go +++ b/test/e2e/ingress/resourcepushing.go @@ -276,4 +276,61 @@ spec: Contains("headers"). Contains("httpbin.com") }) + + ginkgo.It("route priority", func() { + backendSvc, backendSvcPort := s.DefaultHTTPBackend() + apisixRoute := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2alpha1 +kind: ApisixRoute +metadata: + name: httpbin-route +spec: + http: + - name: rule1 + priority: 1 + match: + hosts: + - httpbin.com + paths: + - /ip + backend: + serviceName: %s + servicePort: %d + - name: rule2 + priority: 2 + match: + hosts: + - httpbin.com + paths: + - /ip + exprs: + - subject: + scope: Header + name: X-Foo + op: Equal + value: barbazbar + backend: + serviceName: %s + servicePort: %d + plugins: + - name: request-id + enable: true +`, backendSvc, backendSvcPort[0], backendSvc, backendSvcPort[0]) + + assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute), "creating ApisixRoute") + assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixRoutesCreated(2)) + assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(1)) + + // Hit rule1 + resp := s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.com").Expect() + resp.Status(http.StatusOK) + resp.Body().Contains("origin") + resp.Header("X-Request-Id").Empty() + + // Hit rule2 + resp = s.NewAPISIXClient().GET("/ip").WithHeader("Host", "httpbin.com").WithHeader("X-Foo", "barbazbar").Expect() + resp.Status(http.StatusOK) + resp.Body().Contains("origin") + resp.Header("X-Request-Id").NotEmpty() + }) }) From 5edde00524408f10ae81988b65a04aea8e2269cb Mon Sep 17 00:00:00 2001 From: Alex Zhang Date: Thu, 1 Apr 2021 21:57:22 +0800 Subject: [PATCH 4/5] fix: route rule name cannot be empty and duplicated (#330) --- pkg/kube/translation/apisix_route.go | 8 + pkg/kube/translation/apisix_route_test.go | 173 ++++++++++++++++++++++ 2 files changed, 181 insertions(+) diff --git a/pkg/kube/translation/apisix_route.go b/pkg/kube/translation/apisix_route.go index 52c8364b87..69142938b1 100644 --- a/pkg/kube/translation/apisix_route.go +++ b/pkg/kube/translation/apisix_route.go @@ -100,9 +100,17 @@ func (t *translator) TranslateRouteV2alpha1(ar *configv2alpha1.ApisixRoute) ([]* upstreams []*apisixv1.Upstream ) + ruleNameMap := make(map[string]struct{}) upstreamMap := make(map[string]*apisixv1.Upstream) for _, part := range ar.Spec.HTTP { + if part.Name == "" { + return nil, nil, errors.New("empty route rule name") + } + if _, ok := ruleNameMap[part.Name]; ok { + return nil, nil, errors.New("duplicated route rule name") + } + ruleNameMap[part.Name] = struct{}{} if part.Match == nil { return nil, nil, errors.New("empty route match section") } diff --git a/pkg/kube/translation/apisix_route_test.go b/pkg/kube/translation/apisix_route_test.go index 0a7a80e746..e0a6c59674 100644 --- a/pkg/kube/translation/apisix_route_test.go +++ b/pkg/kube/translation/apisix_route_test.go @@ -15,8 +15,21 @@ package translation import ( + "context" "testing" + fakeapisix "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/clientset/versioned/fake" + apisixinformers "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/informers/externalversions" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "k8s.io/apimachinery/pkg/util/intstr" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/stretchr/testify/assert" configv2alpha1 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2alpha1" @@ -156,3 +169,163 @@ func TestRouteMatchExpr(t *testing.T) { assert.Equal(t, results[8][1].StrVal, "in") assert.Equal(t, results[8][2].SliceVal, []string{"a.com", "b.com"}) } + +func TestTranslateApisixRouteV2alpha1WithEmptyName(t *testing.T) { + ar := &configv2alpha1.ApisixRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ar", + Namespace: "test", + }, + Spec: &configv2alpha1.ApisixRouteSpec{ + HTTP: []*configv2alpha1.ApisixRouteHTTP{ + { + Name: "", + Priority: 0, + }, + }, + }, + } + tr := &translator{} + _, _, err := tr.TranslateRouteV2alpha1(ar) + assert.Equal(t, err.Error(), "empty route rule name") +} + +func TestTranslateApisixRouteV2alpha1WithDuplicatedName(t *testing.T) { + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "svc", + Namespace: "test", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "port1", + Port: 80, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 9080, + }, + }, + { + Name: "port2", + Port: 443, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 9443, + }, + }, + }, + }, + } + endpoints := &corev1.Endpoints{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "svc", + Namespace: "test", + }, + Subsets: []corev1.EndpointSubset{ + { + Ports: []corev1.EndpointPort{ + { + Name: "port1", + Port: 9080, + }, + { + Name: "port2", + Port: 9443, + }, + }, + Addresses: []corev1.EndpointAddress{ + {IP: "192.168.1.1"}, + {IP: "192.168.1.2"}, + }, + }, + }, + } + + client := fake.NewSimpleClientset() + informersFactory := informers.NewSharedInformerFactory(client, 0) + svcInformer := informersFactory.Core().V1().Services().Informer() + svcLister := informersFactory.Core().V1().Services().Lister() + epInformer := informersFactory.Core().V1().Endpoints().Informer() + epLister := informersFactory.Core().V1().Endpoints().Lister() + apisixClient := fakeapisix.NewSimpleClientset() + apisixInformersFactory := apisixinformers.NewSharedInformerFactory(apisixClient, 0) + + _, err := client.CoreV1().Endpoints("test").Create(context.Background(), endpoints, metav1.CreateOptions{}) + assert.Nil(t, err) + _, err = client.CoreV1().Services("test").Create(context.Background(), svc, metav1.CreateOptions{}) + assert.Nil(t, err) + + tr := &translator{ + &TranslatorOptions{ + EndpointsLister: epLister, + ServiceLister: svcLister, + ApisixUpstreamLister: apisixInformersFactory.Apisix().V1().ApisixUpstreams().Lister(), + }, + } + + processCh := make(chan struct{}) + svcInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + processCh <- struct{}{} + }, + }) + epInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + processCh <- struct{}{} + }, + }) + + stopCh := make(chan struct{}) + defer close(stopCh) + go svcInformer.Run(stopCh) + go epInformer.Run(stopCh) + cache.WaitForCacheSync(stopCh, svcInformer.HasSynced) + + <-processCh + <-processCh + + ar := &configv2alpha1.ApisixRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ar", + Namespace: "test", + }, + Spec: &configv2alpha1.ApisixRouteSpec{ + HTTP: []*configv2alpha1.ApisixRouteHTTP{ + { + Name: "rule1", + Match: &configv2alpha1.ApisixRouteHTTPMatch{ + Paths: []string{ + "/*", + }, + }, + Backend: &configv2alpha1.ApisixRouteHTTPBackend{ + ServiceName: "svc", + ServicePort: intstr.IntOrString{ + IntVal: 80, + }, + }, + }, + { + Name: "rule1", + Match: &configv2alpha1.ApisixRouteHTTPMatch{ + Paths: []string{ + "/*", + }, + }, + Backend: &configv2alpha1.ApisixRouteHTTPBackend{ + ServiceName: "svc", + ServicePort: intstr.IntOrString{ + IntVal: 80, + }, + }, + }, + }, + }, + } + + _, _, err = tr.TranslateRouteV2alpha1(ar) + assert.Equal(t, err.Error(), "duplicated route rule name") +} From 80062976ab9bed2c147f1ad7812a003af02e96ff Mon Sep 17 00:00:00 2001 From: Alex Zhang Date: Thu, 1 Apr 2021 22:05:04 +0800 Subject: [PATCH 5/5] chore: use kind to run e2e test suites (#331) * chore: use kind to run e2e test suites * fix * fix: license check failure --- .github/workflows/e2e-test-ci.yml | 19 ++----- Makefile | 64 ++++++++++++++++------- test/e2e/README.md | 16 +++++- test/e2e/scaffold/apisix.go | 63 +--------------------- test/e2e/scaffold/etcd.go | 2 +- test/e2e/scaffold/httpbin.go | 2 +- test/e2e/scaffold/ingress.go | 8 +-- test/e2e/scaffold/k8s.go | 76 +++++++++++++++++---------- test/e2e/scaffold/scaffold.go | 22 ++++---- utils/kind-with-registry.sh | 86 +++++++++++++++++++++++++++++++ 10 files changed, 219 insertions(+), 139 deletions(-) create mode 100755 utils/kind-with-registry.sh diff --git a/.github/workflows/e2e-test-ci.yml b/.github/workflows/e2e-test-ci.yml index ebe21ab4e0..d6f093ec38 100644 --- a/.github/workflows/e2e-test-ci.yml +++ b/.github/workflows/e2e-test-ci.yml @@ -7,25 +7,16 @@ on: pull_request: branches: - master - - ci/e2e jobs: e2e-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Install minikube + - name: Install kind run: | - sh ./utils/minikube.sh - - name: Output cluster info - run: kubectl cluster-info - - name: Add images - run: | - IMAGE_TAG=dev make build-image-to-minikube - eval $(minikube docker-env) - docker pull apache/apisix:dev - docker pull bitnami/etcd:3.4.14-debian-10-r0 - docker pull kennethreitz/httpbin - docker images + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.10.0/kind-linux-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin - name: Setup Go Env uses: actions/setup-go@v1 with: @@ -37,7 +28,7 @@ jobs: - name: Run e2e test cases working-directory: ./ run: | - make e2e-test E2E_SKIP_BUILD=1 E2E_CONCURRENCY=1 + make e2e-test E2E_CONCURRENCY=2 - name: upload coverage profile working-directory: ./test/e2e run: | diff --git a/Makefile b/Makefile index bb04c5e434..0313bbd902 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ default: help VERSION ?= 0.4.0 RELEASE_SRC = apache-apisix-ingress-controller-${VERSION}-src +LOCAL_REGISTRY="localhost:5000" IMAGE_TAG ?= dev GINKGO ?= $(shell which ginkgo) @@ -36,31 +37,37 @@ GO_LDFLAGS ?= "-X=$(VERSYM)=$(VERSION) -X=$(GITSHASYM)=$(GITSHA) -X=$(BUILDOSSYM E2E_CONCURRENCY ?= 1 E2E_SKIP_BUILD ?= 0 -### build: Build apisix-ingress-controller +### build: Build apisix-ingress-controller +.PHONY: build build: go build \ -o apisix-ingress-controller \ -ldflags $(GO_LDFLAGS) \ main.go -### build-image: Build apisix-ingress-controller image +### build-image: Build apisix-ingress-controller image +.PHONY: build-image build-image: docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) . -### lint: Do static lint check +### lint: Do static lint check +.PHONY: lint lint: golangci-lint run -### gofmt: Format all go codes +### gofmt: Format all go codes +.PHONY: gofmt gofmt: find . -type f -name "*.go" | xargs gofmt -w -s -### unit-test: Run unit test cases +### unit-test: Run unit test cases +.PHONY: unit-test unit-test: go test -cover -coverprofile=coverage.txt ./... -### e2e-test: Run e2e test cases (minikube is required) -e2e-test: ginkgo-check build-image-to-minikube +### e2e-test: Run e2e test cases (kind is required) +.PHONY: e2e-test +e2e-test: ginkgo-check push-images-to-kind kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixRoute.yaml kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixUpstream.yaml kubectl apply -f $(PWD)/samples/deploy/crd/v1beta1/ApisixTls.yaml @@ -73,16 +80,38 @@ ifeq ("$(wildcard $(GINKGO))", "") exit 1 endif -# build images to minikube node directly, it's an internal directive, so don't -# expose it's help message. -build-image-to-minikube: +### push-images-to-kind: Push images used in e2e test suites to kind. +.PHONY: push-images-to-kind +push-images-to-kind: kind-up ifeq ($(E2E_SKIP_BUILD), 0) - @minikube version > /dev/null 2>&1 || (echo "ERROR: minikube is required."; exit 1) - @eval $$(minikube docker-env);\ + docker pull apache/apisix:dev + docker tag apache/apisix:dev $(LOCAL_REGISTRY)/apache/apisix:dev + docker push $(LOCAL_REGISTRY)/apache/apisix:dev + + docker pull bitnami/etcd:3.4.14-debian-10-r0 + docker tag bitnami/etcd:3.4.14-debian-10-r0 $(LOCAL_REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0 + docker push $(LOCAL_REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0 + + docker pull kennethreitz/httpbin + docker tag kennethreitz/httpbin $(LOCAL_REGISTRY)/kennethreitz/httpbin + docker push $(LOCAL_REGISTRY)/kennethreitz/httpbin + docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) . + docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(LOCAL_REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG) + docker push $(LOCAL_REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG) endif -### license-check: Do Apache License Header check +### kind-up: Launch a Kubernetes cluster with a image registry by Kind. +.PHONY: kind-up +kind-up: + ./utils/kind-with-registry.sh +### kind-reset: Delete the Kubernetes cluster created by "make kind-up" +.PHONY: kind-reset +kind-reset: + kind delete cluster --name apisix + +### license-check: Do Apache License Header check +.PHONY: license-check license-check: ifeq ("$(wildcard .actions/openwhisk-utilities/scancode/scanCode.py)", "") git clone https://github.com/apache/openwhisk-utilities.git .actions/openwhisk-utilities @@ -90,13 +119,15 @@ ifeq ("$(wildcard .actions/openwhisk-utilities/scancode/scanCode.py)", "") endif .actions/openwhisk-utilities/scancode/scanCode.py --config .actions/ASF-Release.cfg ./ -### help: Show Makefile rules +### help: Show Makefile rules +.PHONY: help help: @echo Makefile rules: @echo @grep -E '^### [-A-Za-z0-9_]+:' Makefile | sed 's/###/ /' -### release-src: Release source +### release-src: Release source +.PHONY: release-src release-src: tar -zcvf $(RELEASE_SRC).tgz \ --exclude .github \ @@ -118,6 +149,3 @@ release-src: mv $(RELEASE_SRC).tgz release/$(RELEASE_SRC).tgz mv $(RELEASE_SRC).tgz.asc release/$(RELEASE_SRC).tgz.asc mv $(RELEASE_SRC).tgz.sha512 release/$(RELEASE_SRC).tgz.sha512 - -.PHONY: build lint help - diff --git a/test/e2e/README.md b/test/e2e/README.md index bc35f93050..c986b7a079 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -20,8 +20,6 @@ apisix ingress controller e2e test suites ========================================= -For running e2e test cases, a Kubernetes cluster is required, [minikube](https://minikube.sigs.k8s.io/docs/start/) is a good choice to build k8s cluster in development environment. - Scaffold --------- @@ -44,3 +42,17 @@ Features -------- Test caes inside `features` directory test some features about APISIX, such as traffic-split, health check and so on. + +Quick Start +----------- + +Run `make e2e-test` to run the e2e test suites in your development environment, a several stuffs that this command will do: + +1. Create a Kubernetes cluster by [kind](https://kind.sigs.k8s.io/), please installing in advance. +2. Build and push all related images to this cluster. +3. Run e2e test suites. + +Step `1` and `2` can be skipped by passing `E2E_SKIP_BUILD=1` to this directive, also, you can customize the +running concurrency of e2e test suites by passing `E2E_CONCURRENCY=X` where `X` is the desired number of cases running in parallel. + +Run `make kind-reset` to delete the cluster that created by `make e2e-test`. diff --git a/test/e2e/scaffold/apisix.go b/test/e2e/scaffold/apisix.go index f4448feba0..5719f45d4d 100644 --- a/test/e2e/scaffold/apisix.go +++ b/test/e2e/scaffold/apisix.go @@ -15,10 +15,7 @@ package scaffold import ( - "errors" "fmt" - "net" - "strconv" "strings" "github.com/gruntwork-io/terratest/modules/k8s" @@ -77,7 +74,7 @@ spec: tcpSocket: port: 9080 timeoutSeconds: 2 - image: "apache/apisix:dev" + image: "localhost:5000/apache/apisix:dev" imagePullPolicy: IfNotPresent name: apisix-deployment-e2e-test ports: @@ -127,63 +124,6 @@ spec: ` ) -func (s *Scaffold) apisixServiceURL() (string, error) { - if len(s.nodes) == 0 { - return "", errors.New("no available node") - } - var addr string - for _, node := range s.nodes { - if len(node.Status.Addresses) > 0 { - addr = node.Status.Addresses[0].Address - break - } - } - for _, port := range s.apisixService.Spec.Ports { - if port.Name == "http" { - return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil - } - } - return "", errors.New("no http port in apisix service") -} - -func (s *Scaffold) apisixServiceHttpsURL() (string, error) { - if len(s.nodes) == 0 { - return "", errors.New("no available node") - } - var addr string - for _, node := range s.nodes { - if len(node.Status.Addresses) > 0 { - addr = node.Status.Addresses[0].Address - break - } - } - for _, port := range s.apisixService.Spec.Ports { - if port.Name == "https" { - return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil - } - } - return "", errors.New("no https port defined in apisix service") -} - -func (s *Scaffold) apisixAdminServiceURL() (string, error) { - if len(s.nodes) == 0 { - return "", errors.New("no available node") - } - var addr string - for _, node := range s.nodes { - if len(node.Status.Addresses) > 0 { - addr = node.Status.Addresses[0].Address - break - } - } - for _, port := range s.apisixService.Spec.Ports { - if port.Name == "http-admin" { - return net.JoinHostPort(addr, strconv.Itoa(int(port.NodePort))), nil - } - } - return "", errors.New("no http-admin port in apisix admin service") -} - func (s *Scaffold) newAPISIX() (*corev1.Service, error) { defaultData, err := s.renderConfig(s.opts.APISIXDefaultConfigPath) if err != nil { @@ -210,6 +150,7 @@ func (s *Scaffold) newAPISIX() (*corev1.Service, error) { if err != nil { return nil, err } + return svc, nil } diff --git a/test/e2e/scaffold/etcd.go b/test/e2e/scaffold/etcd.go index de227e1583..c0559bb69a 100644 --- a/test/e2e/scaffold/etcd.go +++ b/test/e2e/scaffold/etcd.go @@ -65,7 +65,7 @@ spec: tcpSocket: port: 2379 timeoutSeconds: 2 - image: "bitnami/etcd:3.4.14-debian-10-r0" + image: "localhost:5000/bitnami/etcd:3.4.14-debian-10-r0" imagePullPolicy: IfNotPresent name: etcd-deployment-e2e-test ports: diff --git a/test/e2e/scaffold/httpbin.go b/test/e2e/scaffold/httpbin.go index 41701d6ea2..d9fd861942 100644 --- a/test/e2e/scaffold/httpbin.go +++ b/test/e2e/scaffold/httpbin.go @@ -64,7 +64,7 @@ spec: tcpSocket: port: 80 timeoutSeconds: 2 - image: "kennethreitz/httpbin" + image: "localhost:5000/kennethreitz/httpbin" imagePullPolicy: IfNotPresent name: httpbin-deployment-e2e-test ports: diff --git a/test/e2e/scaffold/ingress.go b/test/e2e/scaffold/ingress.go index b3cb70fdb9..67f9343038 100644 --- a/test/e2e/scaffold/ingress.go +++ b/test/e2e/scaffold/ingress.go @@ -227,8 +227,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: "apache/apisix-ingress-controller:dev" - imagePullPolicy: Never + image: "localhost:5000/apache/apisix-ingress-controller:dev" + imagePullPolicy: Always name: ingress-apisix-controller-deployment-e2e-test ports: - containerPort: 8080 @@ -268,11 +268,11 @@ func (s *Scaffold) newIngressAPISIXController() error { if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, crb); err != nil { return err } - s.addFinializer(func() { + s.addFinalizers(func() { err := k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, crb) assert.Nil(s.t, err, "deleting ClusterRoleBinding") }) - s.addFinializer(func() { + s.addFinalizers(func() { err := k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, cr) assert.Nil(s.t, err, "deleting ClusterRole") }) diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go index 0c3fec9c4c..a87a55f36c 100644 --- a/test/e2e/scaffold/k8s.go +++ b/test/e2e/scaffold/k8s.go @@ -109,7 +109,7 @@ func (s *Scaffold) CreateResourceFromStringWithNamespace(yaml, namespace string) defer func() { s.kubectlOptions.Namespace = originalNamespace }() - s.addFinializer(func() { + s.addFinalizers(func() { originalNamespace := s.kubectlOptions.Namespace s.kubectlOptions.Namespace = namespace defer func() { @@ -154,13 +154,9 @@ func ensureNumApisixCRDsCreated(url string, desired int) error { // EnsureNumApisixRoutesCreated waits until desired number of Routes are created in // APISIX cluster. func (s *Scaffold) EnsureNumApisixRoutesCreated(desired int) error { - host, err := s.apisixAdminServiceURL() - if err != nil { - return err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin/routes", } return ensureNumApisixCRDsCreated(u.String(), desired) @@ -169,13 +165,9 @@ func (s *Scaffold) EnsureNumApisixRoutesCreated(desired int) error { // EnsureNumApisixUpstreamsCreated waits until desired number of Upstreams are created in // APISIX cluster. func (s *Scaffold) EnsureNumApisixUpstreamsCreated(desired int) error { - host, err := s.apisixAdminServiceURL() - if err != nil { - return err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin/upstreams", } return ensureNumApisixCRDsCreated(u.String(), desired) @@ -183,13 +175,9 @@ func (s *Scaffold) EnsureNumApisixUpstreamsCreated(desired int) error { // ListApisixUpstreams list all upstreams from APISIX func (s *Scaffold) ListApisixUpstreams() ([]*v1.Upstream, error) { - host, err := s.apisixAdminServiceURL() - if err != nil { - return nil, err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin", } cli, err := apisix.NewClient() @@ -207,13 +195,9 @@ func (s *Scaffold) ListApisixUpstreams() ([]*v1.Upstream, error) { // ListApisixRoutes list all routes from APISIX. func (s *Scaffold) ListApisixRoutes() ([]*v1.Route, error) { - host, err := s.apisixAdminServiceURL() - if err != nil { - return nil, err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin", } cli, err := apisix.NewClient() @@ -231,13 +215,9 @@ func (s *Scaffold) ListApisixRoutes() ([]*v1.Route, error) { // ListApisixTls list all ssl from APISIX func (s *Scaffold) ListApisixTls() ([]*v1.Ssl, error) { - host, err := s.apisixAdminServiceURL() - if err != nil { - return nil, err - } u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixAdminTunnel.Endpoint(), Path: "/apisix/admin", } cli, err := apisix.NewClient() @@ -252,3 +232,47 @@ func (s *Scaffold) ListApisixTls() ([]*v1.Ssl, error) { } return cli.Cluster("").SSL().List(context.TODO()) } + +func (s *Scaffold) newAPISIXTunnels() error { + var ( + adminNodePort int + httpNodePort int + httpsNodePort int + adminPort int + httpPort int + httpsPort int + ) + for _, port := range s.apisixService.Spec.Ports { + if port.Name == "http" { + httpNodePort = int(port.NodePort) + httpPort = int(port.Port) + } else if port.Name == "https" { + httpsNodePort = int(port.NodePort) + httpsPort = int(port.Port) + } else if port.Name == "http-admin" { + adminNodePort = int(port.NodePort) + adminPort = int(port.Port) + } + } + + s.apisixAdminTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test", + adminNodePort, adminPort) + s.apisixHttpTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test", + httpNodePort, httpPort) + s.apisixHttpsTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test", + httpsNodePort, httpsPort) + + if err := s.apisixAdminTunnel.ForwardPortE(s.t); err != nil { + return err + } + s.addFinalizers(s.apisixAdminTunnel.Close) + if err := s.apisixHttpTunnel.ForwardPortE(s.t); err != nil { + return err + } + s.addFinalizers(s.apisixHttpTunnel.Close) + if err := s.apisixHttpsTunnel.ForwardPortE(s.t); err != nil { + return err + } + s.addFinalizers(s.apisixHttpsTunnel.Close) + return nil +} diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 7a9da7e6ef..d64e07d28e 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -62,6 +62,10 @@ type Scaffold struct { httpbinService *corev1.Service finializers []func() + apisixAdminTunnel *k8s.Tunnel + apisixHttpTunnel *k8s.Tunnel + apisixHttpsTunnel *k8s.Tunnel + // Used for template rendering. EtcdServiceFQDN string } @@ -137,18 +141,11 @@ func (s *Scaffold) DefaultHTTPBackend() (string, []int32) { return s.httpbinService.Name, ports } -// GetAPISIXEndpoint returns the service and port (as an endpoint). -func (s *Scaffold) GetAPISIXEndpoint() (string, error) { - return s.apisixServiceURL() -} - // NewAPISIXClient creates the default HTTP client. func (s *Scaffold) NewAPISIXClient() *httpexpect.Expect { - host, err := s.apisixServiceURL() - assert.Nil(s.t, err, "getting apisix service url") u := url.URL{ Scheme: "http", - Host: host, + Host: s.apisixHttpTunnel.Endpoint(), } return httpexpect.WithConfig(httpexpect.Config{ BaseURL: u.String(), @@ -166,11 +163,9 @@ func (s *Scaffold) NewAPISIXClient() *httpexpect.Expect { // NewAPISIXHttpsClient creates the default HTTPs client. func (s *Scaffold) NewAPISIXHttpsClient() *httpexpect.Expect { - host, err := s.apisixServiceHttpsURL() - assert.Nil(s.t, err, "getting apisix service url") u := url.URL{ Scheme: "https", - Host: host, + Host: s.apisixHttpsTunnel.Endpoint(), } return httpexpect.WithConfig(httpexpect.Config{ BaseURL: u.String(), @@ -213,6 +208,9 @@ func (s *Scaffold) beforeEach() { err = s.waitAllAPISIXPodsAvailable() assert.Nil(s.t, err, "waiting for apisix ready") + err = s.newAPISIXTunnels() + assert.Nil(s.t, err, "creating apisix tunnels") + s.httpbinService, err = s.newHTTPBIN() assert.Nil(s.t, err, "initializing httpbin") @@ -239,7 +237,7 @@ func (s *Scaffold) afterEach() { time.Sleep(3 * time.Second) } -func (s *Scaffold) addFinializer(f func()) { +func (s *Scaffold) addFinalizers(f func()) { s.finializers = append(s.finializers, f) } diff --git a/utils/kind-with-registry.sh b/utils/kind-with-registry.sh new file mode 100755 index 0000000000..ea01a2c6bb --- /dev/null +++ b/utils/kind-with-registry.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -o errexit +set -o nounset +set -o pipefail + +# desired cluster name; default is "apisix" +KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-apisix}" + +if kind get clusters | grep -q ^apisix$ ; then + echo "cluster already exists, moving on" + exit 0 +fi + +# create registry container unless it already exists +kind_version=$(kind version) +kind_network='kind' +reg_name='kind-registry' +reg_port='5000' +case "${kind_version}" in + "kind v0.7."* | "kind v0.6."* | "kind v0.5."*) + kind_network='bridge' + ;; +esac + +# create registry container unless it already exists +running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" +if [ "${running}" != 'true' ]; then + docker run \ + -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +reg_host="${reg_name}" +if [ "${kind_network}" = "bridge" ]; then + reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" +fi +echo "Registry Host: ${reg_host}" + +# create a cluster with the local registry enabled in containerd +cat <