Skip to content

Commit

Permalink
feat: support custom registry for e2e test (#896)
Browse files Browse the repository at this point in the history 
* feat: support custom registry for e2e test

* fix: add README and format variable CUSTOM_REGISTRT in makefile

* fix: format var name

* fix: rename LOCAL_REGISTRY variable name

* fix: update registry name in code

* fix: remove unused kind up part

* fix: format Makefile
  • Loading branch information
@Belyenochi
Belyenochi committed Mar 8, 2022
1 parent 1ddbfa6 commit f84a083
Show file tree
Hide file tree
Showing 8 changed files with 47 additions and 30 deletions.
45 changes: 25 additions & 20 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ default: help

VERSION ?= 1.4.0
RELEASE_SRC = apache-apisix-ingress-controller-${VERSION}-src
LOCAL_REGISTRY="localhost:5000"
REGISTRY ?="localhost:5000"
IMAGE_TAG ?= dev

GITSHA ?= "no-git-module"
Expand Down Expand Up @@ -63,52 +63,57 @@ lint:
unit-test:
go test -cover -coverprofile=coverage.txt ./...

### e2e-test: Run e2e test cases (kind is required)
### e2e-test: Run e2e test cases (in existing clusters directly)
.PHONY: e2e-test
e2e-test: ginkgo-check push-images-to-kind
e2e-test: ginkgo-check push-images
kubectl apply -k $(PWD)/samples/deploy/crd
cd test/e2e \
&& go mod download \
&& export REGISTRY=$(REGISTRY) \
&& ACK_GINKGO_RC=true ginkgo -cover -coverprofile=coverage.txt -r --randomizeSuites --randomizeAllSpecs --trace --nodes=$(E2E_CONCURRENCY)

### e2e-test-local: Run e2e test cases (kind is required)
.PHONY: e2e-test-local
e2e-test-local: kind-up e2e-test

.PHONY: ginkgo-check
ginkgo-check:
ifeq ("$(wildcard $(GINKGO))", "")
@echo "ERROR: Need to install ginkgo first, run: go get -u github.com/onsi/ginkgo/ginkgo"
exit 1
endif

### push-images-to-kind: Push images used in e2e test suites to kind.
.PHONY: push-images-to-kind
push-images-to-kind: kind-up
### push-images: Push images used in e2e test suites to kind or custom registry.
.PHONY: push-images
push-images:
ifeq ($(E2E_SKIP_BUILD), 0)
docker pull apache/apisix:2.12.0-alpine
docker tag apache/apisix:2.12.0-alpine $(LOCAL_REGISTRY)/apache/apisix:dev
docker push $(LOCAL_REGISTRY)/apache/apisix:dev
docker tag apache/apisix:2.12.0-alpine $(REGISTRY)/apache/apisix:dev
docker push $(REGISTRY)/apache/apisix:dev

docker pull bitnami/etcd:3.4.14-debian-10-r0
docker tag bitnami/etcd:3.4.14-debian-10-r0 $(LOCAL_REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0
docker push $(LOCAL_REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0
docker tag bitnami/etcd:3.4.14-debian-10-r0 $(REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0
docker push $(REGISTRY)/bitnami/etcd:3.4.14-debian-10-r0

docker pull kennethreitz/httpbin
docker tag kennethreitz/httpbin $(LOCAL_REGISTRY)/kennethreitz/httpbin
docker push $(LOCAL_REGISTRY)/kennethreitz/httpbin
docker tag kennethreitz/httpbin $(REGISTRY)/kennethreitz/httpbin
docker push $(REGISTRY)/kennethreitz/httpbin

docker build -t test-backend:$(IMAGE_TAG) --build-arg ENABLE_PROXY=true ./test/e2e/testbackend
docker tag test-backend:$(IMAGE_TAG) $(LOCAL_REGISTRY)/test-backend:$(IMAGE_TAG)
docker push $(LOCAL_REGISTRY)/test-backend:$(IMAGE_TAG)
docker tag test-backend:$(IMAGE_TAG) $(REGISTRY)/test-backend:$(IMAGE_TAG)
docker push $(REGISTRY)/test-backend:$(IMAGE_TAG)

docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) --build-arg ENABLE_PROXY=true .
docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(LOCAL_REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
docker push $(LOCAL_REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
docker push $(REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)

docker pull jmalloc/echo-server:latest
docker tag jmalloc/echo-server:latest $(LOCAL_REGISTRY)/jmalloc/echo-server:latest
docker push $(LOCAL_REGISTRY)/jmalloc/echo-server:latest
docker tag jmalloc/echo-server:latest $(REGISTRY)/jmalloc/echo-server:latest
docker push $(REGISTRY)/jmalloc/echo-server:latest

docker pull busybox:1.28
docker tag busybox:1.28 $(LOCAL_REGISTRY)/busybox:1.28
docker push $(LOCAL_REGISTRY)/busybox:1.28
docker tag busybox:1.28 $(REGISTRY)/busybox:1.28
docker push $(REGISTRY)/busybox:1.28
endif

### kind-up: Launch a Kubernetes cluster with a image registry by Kind.
Expand Down
4 changes: 3 additions & 1 deletion test/e2e/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,14 @@ Test caes inside `features` directory test some features about APISIX, such as t
Quick Start
-----------

Run `make e2e-test` to run the e2e test suites in your development environment, a several stuffs that this command will do:
Run `make e2e-test-local` to run the e2e test suites in your development environment, a several stuffs that this command will do:

1. Create a Kubernetes cluster by [kind](https://kind.sigs.k8s.io/), please installing in advance.
2. Build and push all related images to this cluster.
3. Run e2e test suites.

Run `make e2e-test` to run the e2e test suites in an existing cluster, you can specify custom registry by passing REGISTRY(eg docker.io).

Step `1` and `2` can be skipped by passing `E2E_SKIP_BUILD=1` to this directive, also, you can customize the
running concurrency of e2e test suites by passing `E2E_CONCURRENCY=X` where `X` is the desired number of cases running in parallel.

Expand Down
2 changes: 1 addition & 1 deletion test/e2e/scaffold/apisix.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ func (s *Scaffold) newAPISIX() (*corev1.Service, error) {
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, configData); err != nil {
return nil, err
}
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, _apisixDeployment); err != nil {
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, s.FormatRegistry(_apisixDeployment)); err != nil {
return nil, err
}
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, _apisixService); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/scaffold/etcd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ spec:
)

func (s *Scaffold) newEtcd() (*corev1.Service, error) {
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, etcdDeployment); err != nil {
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, s.FormatRegistry(etcdDeployment)); err != nil {
return nil, err
}
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, etcdService); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/scaffold/httpbin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ spec:
)

func (s *Scaffold) newHTTPBIN() (*corev1.Service, error) {
httpbinDeployment := fmt.Sprintf(_httpbinDeploymentTemplate, 1)
httpbinDeployment := fmt.Sprintf(s.FormatRegistry(_httpbinDeploymentTemplate), 1)
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, httpbinDeployment); err != nil {
return nil, err
}
Expand All @@ -105,7 +105,7 @@ func (s *Scaffold) newHTTPBIN() (*corev1.Service, error) {

// ScaleHTTPBIN scales the number of HTTPBIN pods to desired.
func (s *Scaffold) ScaleHTTPBIN(desired int) error {
httpbinDeployment := fmt.Sprintf(_httpbinDeploymentTemplate, desired)
httpbinDeployment := fmt.Sprintf(s.FormatRegistry(_httpbinDeploymentTemplate), desired)
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, httpbinDeployment); err != nil {
return err
}
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/scaffold/ingress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,9 +407,9 @@ func (s *Scaffold) newIngressAPISIXController() error {
var ingressAPISIXDeployment string
label := fmt.Sprintf("apisix.ingress.watch=%s", s.namespace)
if s.opts.EnableWebhooks {
ingressAPISIXDeployment = fmt.Sprintf(_ingressAPISIXDeploymentTemplate, s.opts.IngressAPISIXReplicas, s.namespace, label, s.opts.APISIXRouteVersion, s.opts.APISIXPublishAddress, _volumeMounts, _webhookCertSecret)
ingressAPISIXDeployment = fmt.Sprintf(s.FormatRegistry(_ingressAPISIXDeploymentTemplate), s.opts.IngressAPISIXReplicas, s.namespace, label, s.opts.APISIXRouteVersion, s.opts.APISIXPublishAddress, _volumeMounts, _webhookCertSecret)
} else {
ingressAPISIXDeployment = fmt.Sprintf(_ingressAPISIXDeploymentTemplate, s.opts.IngressAPISIXReplicas, s.namespace, label, s.opts.APISIXRouteVersion, s.opts.APISIXPublishAddress, "", _webhookCertSecret)
ingressAPISIXDeployment = fmt.Sprintf(s.FormatRegistry(_ingressAPISIXDeploymentTemplate), s.opts.IngressAPISIXReplicas, s.namespace, label, s.opts.APISIXRouteVersion, s.opts.APISIXPublishAddress, "", _webhookCertSecret)
}

err = k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, ingressAPISIXDeployment)
Expand Down Expand Up @@ -515,9 +515,9 @@ func (s *Scaffold) ScaleIngressController(desired int) error {
var ingressDeployment string
label := fmt.Sprintf("apisix.ingress.watch=%s", s.namespace)
if s.opts.EnableWebhooks {
ingressDeployment = fmt.Sprintf(_ingressAPISIXDeploymentTemplate, desired, s.namespace, label, s.opts.APISIXRouteVersion, s.opts.APISIXPublishAddress, _volumeMounts, _webhookCertSecret)
ingressDeployment = fmt.Sprintf(s.FormatRegistry(_ingressAPISIXDeploymentTemplate), desired, s.namespace, label, s.opts.APISIXRouteVersion, s.opts.APISIXPublishAddress, _volumeMounts, _webhookCertSecret)
} else {
ingressDeployment = fmt.Sprintf(_ingressAPISIXDeploymentTemplate, desired, s.namespace, label, s.opts.APISIXRouteVersion, s.opts.APISIXPublishAddress, "", _webhookCertSecret)
ingressDeployment = fmt.Sprintf(s.FormatRegistry(_ingressAPISIXDeploymentTemplate), desired, s.namespace, label, s.opts.APISIXRouteVersion, s.opts.APISIXPublishAddress, "", _webhookCertSecret)
}
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, ingressDeployment); err != nil {
return err
Expand Down
10 changes: 10 additions & 0 deletions test/e2e/scaffold/scaffold.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,16 @@ func (s *Scaffold) renderConfig(path string) (string, error) {
return buf.String(), nil
}

// FormatRegistry replace default registry to custom registry if exist
func (s *Scaffold) FormatRegistry(workloadTemplate string) string {
customRegistry, isExist := os.LookupEnv("REGISTRY")
if isExist {
return strings.Replace(workloadTemplate, "localhost:5000", customRegistry, -1)
} else {
return workloadTemplate
}
}

func waitExponentialBackoff(condFunc func() (bool, error)) error {
backoff := wait.Backoff{
Duration: 500 * time.Millisecond,
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/scaffold/test_backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ spec:
)

func (s *Scaffold) newTestBackend() (*corev1.Service, error) {
backendDeployment := fmt.Sprintf(_testBackendDeploymentTemplate, 1)
backendDeployment := fmt.Sprintf(s.FormatRegistry(_testBackendDeploymentTemplate), 1)
if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, backendDeployment); err != nil {
return nil, err
}
Expand Down

0 comments on commit f84a083

Please sign in to comment.