Enable e2e tests using emulators for cloud providers [ aws, gcp, azure ]

.ci/integration_test

@@ -125,8 +125,11 @@ function create_etcd_data_directory() {
   mkdir -p ${ETCD_DATA_DIR}
 }
 
-function get_aws_existing_region() {
+function get_aws_existing_credentials() {
   export REGION=`cat ${HOME}/.aws/config | grep -e "^.*region.*$" | sed "s/^.*region[ ]*=[ ]*//"`
+  export AWS_DEFAULT_REGION=${REGION}
+  export AWS_ACCESS_KEY_ID=`cat ${HOME}/.aws/credentials | grep -e "^.*aws_access_key_id.*$" | sed "s/^.*aws_access_key_id[ ]*=[ ]*//"`
+  export AWS_SECRET_ACCESS_KEY=`cat ${HOME}/.aws/credentials | grep -e "^.*aws_secret_access_key.*$" | sed "s/^.*aws_secret_access_key[ ]*=[ ]*//"`
 }
 
 #############################
@@ -155,6 +158,9 @@ EOF
 }
 EOF
   export AWS_APPLICATION_CREDENTIALS_JSON="${credentials_file}"
+  export AWS_ACCESS_KEY_ID=$1
+  export AWS_SECRET_ACCESS_KEY=$2
+  export AWS_DEFAULT_REGION=$3 
 }
 
 function create_aws_secret() {
@@ -192,7 +198,7 @@ function setup-aws-infrastructure() {
   if [[ "${USE_EXISTING_AWS_SECRET}" == "1" ]]; then
     create_aws_secret
   else
-    get_aws_existing_region
+    get_aws_existing_credentials
   fi
   create_s3_bucket
   echo "AWS infrastructure setup completed."
@@ -256,7 +262,7 @@ function run_test_as_processes() {
   setup_test_cluster
 
   echo "Starting integration tests..."
-  cd test/e2e/integration
+  cd test/integration
 
   set +e
   ginkgo -r -mod=vendor
@@ -287,18 +293,18 @@ function run_test_on_cluster() {
     setup-aws-infrastructure
   fi
 
-  export ETCD_VERSION=${ETCD_VERSION:-"v3.4.13-bootstrap-1"}
+  export ETCD_VERSION=${ETCD_VERSION:-"v0.1.1"}
   echo "Etcd version: ${ETCD_VERSION}"
 
-  export ETCDBR_VERSION=${ETCDBR_VERSION:-${ETCDBR_VER:-"v0.12.1"}}
+  export ETCDBR_VERSION=${ETCDBR_VERSION:-${ETCDBR_VER:-"v0.29.0-dev"}}
   echo "Etcd-backup-restore version: ${ETCDBR_VERSION}"
 
   echo "Starting integration tests on k8s cluster."
 
   set +e
 
   if [ -r "$INTEGRATION_TEST_KUBECONFIG" ]; then
-    KUBECONFIG=$INTEGRATION_TEST_KUBECONFIG STORAGE_CONTAINER=$TEST_ID ginkgo -v -timeout=15m -mod=vendor test/e2e/integrationcluster
+    KUBECONFIG=$INTEGRATION_TEST_KUBECONFIG STORAGE_CONTAINER=$TEST_ID PROVIDER=aws ginkgo -v -timeout=15m -mod=vendor test/integrationcluster
     TEST_RESULT=$?
     echo "Successfully completed all tests."
   else

.gitignore

@@ -17,7 +17,8 @@
 .glide/
 
 # Build Binaries
-bin
+/bin
+/hack/tools/bin
 test/output
 test/e2e_test_data
 default.bkp*
@@ -28,6 +29,9 @@ compctedsnap.bkp*
 .vscode
 .idea/
 
+# kubeconfig
+hack/e2e-test/infrastructure/kind/kubeconfig
+
 # developers workspace
 tmp
 dev

Makefile

@@ -3,17 +3,22 @@
 # SPDX-License-Identifier: Apache-2.0
 
 VERSION             ?= $(shell cat VERSION)
+REPO_ROOT           := $(shell dirname "$(realpath $(lastword $(MAKEFILE_LIST)))")
 REGISTRY            ?= europe-docker.pkg.dev/gardener-project/snapshots
 IMAGE_REPOSITORY    := $(REGISTRY)/gardener/etcdbrctl
 IMAGE_TAG           := $(VERSION)
 BUILD_DIR           := build
 BIN_DIR             := bin
 COVERPROFILE        := test/output/coverprofile.out
+KUBECONFIG_PATH     :=$(REPO_ROOT)/hack/e2e-test/infrastructure/kind/kubeconfig
 
 IMG ?= ${IMAGE_REPOSITORY}:${IMAGE_TAG}
 
 .DEFAULT_GOAL := build-local
 
+TOOLS_DIR := hack/tools
+include $(REPO_ROOT)/hack/tools.mk
+
 .PHONY: revendor
 revendor:
 	@env GO111MODULE=on go mod tidy -v
@@ -24,6 +29,8 @@ update-dependencies:
 	@env GO111MODULE=on go get -u
 	@make revendor
 
+kind-up kind-down ci-e2e-kind deploy-localstack deploy-fakegcs deploy-azurite test-e2e: export KUBECONFIG = $(KUBECONFIG_PATH)
+
 .PHONY: build
 build:
 	@.ci/build
@@ -65,10 +72,46 @@ integration-test:
 	@.ci/integration_test
 
 .PHONY: integration-test-cluster
-integration-test-cluster:
+integration-test-cluster: $(KIND) $(HELM) $(GINKGO) $(KUBECTL)
 	@.ci/integration_test cluster
 
 .PHONY: show-coverage
 show-coverage:
 	@if [ ! -f $(COVERPROFILE) ]; then echo "$(COVERPROFILE) is not yet built. Please run 'COVER=true make test'"; false; fi
 	@go tool cover -html $(COVERPROFILE)
+
+.PHONY: test-e2e
+test-e2e: $(KIND) $(HELM) $(GINKGO) $(KUBECTL)
+	@"$(REPO_ROOT)/hack/e2e-test/run-e2e-test.sh" $(PROVIDERS)
+
+.PHONY: kind-up
+kind-up: $(KIND)
+	./hack/kind-up.sh
+
+.PHONY: kind-down
+kind-down: $(KIND)
+	kind delete cluster --name etcdbr-e2e
+
+.PHONY: deploy-localstack
+deploy-localstack: $(KUBECTL)
+	./hack/deploy-localstack.sh
+
+.PHONY: deploy-fakegcs
+deploy-fakegcs: $(KUBECTL)
+	./hack/deploy-fakegcs.sh
+
+.PHONY: deploy-azurite
+deploy-azurite: $(KUBECTL)
+	./hack/deploy-azurite.sh
+
+.PHONY: ci-e2e-kind
+ci-e2e-kind:
+	./hack/ci-e2e-kind.sh $(PROVIDERS)
+
+.PHONY: pr-test-e2e
+pr-test-e2e:
+	./hack/ci-e2e-kind.sh aws
+
+.PHONY: merge-test-e2e
+merge-test-e2e:
+	./hack/ci-e2e-kind.sh aws,gcp,azure

chart/etcd-backup-restore/templates/etcd-backup-secret.yaml

@@ -14,6 +14,12 @@ data:
   region: {{ .Values.backup.s3.region | b64enc }}
   secretAccessKey: {{ .Values.backup.s3.secretAccessKey | b64enc  }}
   accessKeyID: {{ .Values.backup.s3.accessKeyID | b64enc }}
+  {{- if .Values.backup.s3.endpoint }}
+  endpoint: {{ .Values.backup.s3.endpoint | b64enc }}
+  {{- end }}
+  {{- if .Values.backup.s3.s3ForcePathStyle }}
+  s3ForcePathStyle: {{ .Values.backup.s3.s3ForcePathStyle | b64enc }}
+  {{- end }}
 {{- else if eq .Values.backup.storageProvider "ABS" }}
   storageAccount: {{ .Values.backup.abs.storageAccount | b64enc }}
   storageKey : {{ .Values.backup.abs.storageKey | b64enc }}

chart/etcd-backup-restore/templates/etcd-client-service.yaml

@@ -18,3 +18,11 @@ spec:
     protocol: TCP
     port: {{ .Values.servicePorts.client }}
     targetPort: {{ .Values.servicePorts.client }}
+  - name: server
+    protocol: TCP
+    port: {{ .Values.servicePorts.server }}
+    targetPort: {{ .Values.servicePorts.server }}
+  - name: backuprestore
+    protocol: TCP
+    port: {{ .Values.servicePorts.backupRestore }}
+    targetPort: {{ .Values.servicePorts.backupRestore }}

chart/etcd-backup-restore/templates/etcd-configmap.yaml

@@ -22,6 +22,8 @@ data:
     # Number of committed transactions to trigger a snapshot to disk.
     snapshot-count: 75000
 
+    enable-v2: false
+
     # Raise alarms when backend size exceeds the given quota. 0 means use the
     # default quota.
     {{- if .Values.backup.etcdQuotaBytes }}
@@ -31,12 +33,24 @@ data:
     # List of comma separated URLs to listen on for client traffic.
     listen-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }}
 
+    # List of comma separated URLs to listen on for peer traffic.
+    listen-peer-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.server }}
+
     # List of this member's client URLs to advertise to the public.
     # The URLs needed to be a comma-separated list.
-    advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }}
+    advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}@{{ .Release.Name }}-etcd-peer@{{ .Release.Namespace }}@{{ .Values.servicePorts.client }}
+    # advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }}
+
+
+    # List of this member's peer URLs to advertise to the public.
+    # The URLs needed to be a comma-separated list.
+    initial-advertise-peer-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}@{{ .Release.Name }}-etcd-peer@{{ .Release.Namespace }}@{{ .Values.servicePorts.server }}
+
+    # List of server endpoints with which this cluster should be started
+    initial-cluster: {{ .Release.Name }}-etcd-0={{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://{{ .Release.Name }}-etcd-0.{{ .Release.Name }}-etcd-peer.{{ .Release.Namespace }}.svc:{{ .Values.servicePorts.server }}
 
     # Initial cluster token for the etcd cluster during bootstrap.
-    initial-cluster-token: 'new'
+    initial-cluster-token: 'etcd-cluster'
 
     # Initial cluster state ('new' or 'existing').
     initial-cluster-state: 'new'

chart/etcd-backup-restore/templates/etcd-peer-service.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-etcd-peer
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: etcd
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  publishNotReadyAddresses: true
+  type: ClusterIP
+  clusterIP: None
+  clusterIPs:
+    - None
+  internalTrafficPolicy: Cluster
+  imFamilyPolicy: SingleStack
+  sessionAffinity: None
+  selector:
+    app.kubernetes.io/name: etcd
+    app.kubernetes.io/instance: {{ .Release.Name }}
+  ports:
+  - name: peer
+    protocol: TCP
+    port: {{ .Values.servicePorts.server }}
+    targetPort: {{ .Values.servicePorts.server }}