Enable e2e tests with cloud provider emulators for aws,gcp,azure

gardener · May 29, 2024 · dd4a097 · dd4a097
1 parent c1f0bbe
commit dd4a097
Show file tree

Hide file tree

Showing 25 changed files with 1,162 additions and 217 deletions.
diff --git a/.ci/integration_test b/.ci/integration_test
@@ -155,6 +155,9 @@ EOF
 }
 EOF
   export AWS_APPLICATION_CREDENTIALS_JSON="${credentials_file}"
+  export AWS_ACCESS_KEY_ID=$1
+  export AWS_SECRET_ACCESS_KEY=$2
+  export AWS_DEFAULT_REGION=$3 
 }
 
 function create_aws_secret() {
@@ -171,6 +174,7 @@ function create_aws_secret() {
 
 function delete_aws_secret() {
   rm -rf ${HOME}/.aws
+  unset AWS_APPLICATION_CREDENTIALS_JSON AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION
 }
 
 function create_s3_bucket() {
@@ -256,7 +260,7 @@ function run_test_as_processes() {
   setup_test_cluster
 
   echo "Starting integration tests..."
-  cd test/e2e/integration
+  cd test/integration
 
   set +e
   ginkgo -r -mod=vendor
@@ -290,15 +294,16 @@ function run_test_on_cluster() {
   export ETCD_VERSION=${ETCD_VERSION:-"v3.4.13-bootstrap-1"}
   echo "Etcd version: ${ETCD_VERSION}"
 
-  export ETCDBR_VERSION=${ETCDBR_VERSION:-${ETCDBR_VER:-"v0.12.1"}}
+  export ETCDBR_VERSION=${ETCDBR_VERSION:-${ETCDBR_VER:-"v0.24.7"}}
   echo "Etcd-backup-restore version: ${ETCDBR_VERSION}"
 
   echo "Starting integration tests on k8s cluster."
 
   set +e
 
   if [ -r "$INTEGRATION_TEST_KUBECONFIG" ]; then
-    KUBECONFIG=$INTEGRATION_TEST_KUBECONFIG STORAGE_CONTAINER=$TEST_ID ginkgo -v -timeout=15m -mod=vendor test/e2e/integrationcluster
+    #  Set the PROVIDER env variable to aws for the integration tests
+    KUBECONFIG=$INTEGRATION_TEST_KUBECONFIG STORAGE_CONTAINER=$TEST_ID PROVIDER=aws ginkgo -v -timeout=15m -mod=vendor test/e2e/integrationcluster
     TEST_RESULT=$?
     echo "Successfully completed all tests."
   else

diff --git a/Makefile b/Makefile
@@ -3,12 +3,14 @@
 # SPDX-License-Identifier: Apache-2.0
 
 VERSION             ?= $(shell cat VERSION)
+REPO_ROOT           := $(shell dirname "$(realpath $(lastword $(MAKEFILE_LIST)))")
 REGISTRY            ?= europe-docker.pkg.dev/gardener-project/snapshots
 IMAGE_REPOSITORY    := $(REGISTRY)/gardener/etcdbrctl
 IMAGE_TAG           := $(VERSION)
 BUILD_DIR           := build
 BIN_DIR             := bin
 COVERPROFILE        := test/output/coverprofile.out
+KUBECONFIG_PATH     :=$(REPO_ROOT)/hack/e2e-test/infrastructure/kind/kubeconfig
 
 IMG ?= ${IMAGE_REPOSITORY}:${IMAGE_TAG}
 
@@ -24,6 +26,8 @@ update-dependencies:
 	@env GO111MODULE=on go get -u
 	@make revendor
 
+kind-up kind-down ci-e2e-kind deploy-localstack test-e2e: export KUBECONFIG = $(KUBECONFIG_PATH)
+
 .PHONY: build
 build:
 	@.ci/build
@@ -72,3 +76,39 @@ integration-test-cluster:
 show-coverage:
 	@if [ ! -f $(COVERPROFILE) ]; then echo "$(COVERPROFILE) is not yet built. Please run 'COVER=true make test'"; false; fi
 	@go tool cover -html $(COVERPROFILE)
+
+.PHONY: test-e2e
+test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD)
+	@"$(REPO_ROOT)/hack/e2e-test/run-e2e-test.sh" $(PROVIDERS)
+
+.PHONY: kind-up
+kind-up: $(KIND)
+	./hack/kind-up.sh
+
+.PHONY: kind-down
+kind-down: $(KIND)
+	$(KIND) delete cluster --name etcdbr-e2e
+
+.PHONY: deploy-localstack
+deploy-localstack: $(KUBECTL)
+	./hack/deploy-localstack.sh
+
+.PHONY: deploy-fakegcs
+deploy-fakegcs: $(KUBECTL)
+	./hack/deploy-fakegcs.sh
+
+.PHONY: deploy-azurite
+deploy-azurite: $(KUBECTL)
+	./hack/deploy-azurite.sh
+
+.PHONY: ci-e2e-kind
+ci-e2e-kind:
+	./hack/ci-e2e-kind.sh
+
+.PHONY: pr-test-e2e
+pr-test-e2e:
+	./hack/ci-e2e-kind.sh aws
+
+.PHONY: merge-test-e2e
+merge-test-e2e:
+	./hack/ci-e2e-kind.sh aws,gcp,azure
diff --git a/chart/etcd-backup-restore/templates/etcd-backup-secret.yaml b/chart/etcd-backup-restore/templates/etcd-backup-secret.yaml
@@ -14,6 +14,12 @@ data:
   region: {{ .Values.backup.s3.region | b64enc }}
   secretAccessKey: {{ .Values.backup.s3.secretAccessKey | b64enc  }}
   accessKeyID: {{ .Values.backup.s3.accessKeyID | b64enc }}
+  {{- if .Values.backup.s3.endpoint }}
+  endpoint: {{ .Values.backup.s3.endpoint | b64enc }}
+  {{- end }}
+  {{- if .Values.backup.s3.s3ForcePathStyle }}
+  s3ForcePathStyle: {{ .Values.backup.s3.s3ForcePathStyle | b64enc }}
+  {{- end }}
 {{- else if eq .Values.backup.storageProvider "ABS" }}
   storageAccount: {{ .Values.backup.abs.storageAccount | b64enc }}
   storageKey : {{ .Values.backup.abs.storageKey | b64enc }}
@@ -24,7 +30,7 @@ data:
   storageAPIEndpoint: {{ .Values.backup.abs.storageAPIEndpoint | b64enc}}
   {{- end }}
 {{- else if eq .Values.backup.storageProvider "GCS" }}
-  serviceaccount.json : {{ .Values.backup.gcs.serviceAccountJson | b64enc }}
+  serviceaccount.json : {{ .Values.backup.gcs.serviceAccountJson | b64enc}}
   {{- if .Values.backup.gcs.storageAPIEndpoint }}
   storageAPIEndpoint: {{ .Values.backup.gcs.storageAPIEndpoint | b64enc}}
   {{- end }}

diff --git a/chart/etcd-backup-restore/templates/etcd-client-service.yaml b/chart/etcd-backup-restore/templates/etcd-client-service.yaml
@@ -18,3 +18,11 @@ spec:
     protocol: TCP
     port: {{ .Values.servicePorts.client }}
     targetPort: {{ .Values.servicePorts.client }}
+  - name: server
+    protocol: TCP
+    port: {{ .Values.servicePorts.server }}
+    targetPort: {{ .Values.servicePorts.server }}
+  - name: backuprestore
+    protocol: TCP
+    port: {{ .Values.servicePorts.backupRestore }}
+    targetPort: {{ .Values.servicePorts.backupRestore }}
diff --git a/chart/etcd-backup-restore/templates/etcd-configmap.yaml b/chart/etcd-backup-restore/templates/etcd-configmap.yaml
@@ -22,6 +22,8 @@ data:
     # Number of committed transactions to trigger a snapshot to disk.
     snapshot-count: 75000
 
+    enable-v2: false
+
     # Raise alarms when backend size exceeds the given quota. 0 means use the
     # default quota.
     {{- if .Values.backup.etcdQuotaBytes }}
@@ -31,12 +33,24 @@ data:
     # List of comma separated URLs to listen on for client traffic.
     listen-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }}
 
+    # List of comma separated URLs to listen on for peer traffic.
+    listen-peer-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.server }}
+
     # List of this member's client URLs to advertise to the public.
     # The URLs needed to be a comma-separated list.
-    advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }}
+    advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}@{{ .Release.Name }}-etcd-peer@{{ .Release.Namespace }}@{{ .Values.servicePorts.client }}
+    # advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }}
+
+
+    # List of this member's peer URLs to advertise to the public.
+    # The URLs needed to be a comma-separated list.
+    initial-advertise-peer-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}@{{ .Release.Name }}-etcd-peer@{{ .Release.Namespace }}@{{ .Values.servicePorts.server }}
+
+    # List of server endpoints with which this cluster should be started
+    initial-cluster: {{ .Release.Name }}-etcd-0={{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://{{ .Release.Name }}-etcd-0.{{ .Release.Name }}-etcd-peer.{{ .Release.Namespace }}.svc:{{ .Values.servicePorts.server }}
 
     # Initial cluster token for the etcd cluster during bootstrap.
-    initial-cluster-token: 'new'
+    initial-cluster-token: 'etcd-cluster'
 
     # Initial cluster state ('new' or 'existing').
     initial-cluster-state: 'new'

diff --git a/chart/etcd-backup-restore/templates/etcd-peer-service.yaml b/chart/etcd-backup-restore/templates/etcd-peer-service.yaml
@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-etcd-peer
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: etcd
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  publishNotReadyAddresses: true
+  type: ClusterIP
+  clusterIP: None
+  clusterIPs:
+    - None
+  sessionAffinity: None
+  selector:
+    app.kubernetes.io/name: etcd
+    app.kubernetes.io/instance: {{ .Release.Name }}
+  ports:
+  - name: peer
+    protocol: TCP
+    port: {{ .Values.servicePorts.server }}
+    targetPort: {{ .Values.servicePorts.server }}
diff --git a/chart/etcd-backup-restore/templates/etcd-statefulset.yaml b/chart/etcd-backup-restore/templates/etcd-statefulset.yaml
@@ -27,41 +27,42 @@ spec:
         app.kubernetes.io/name: etcd
         app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
+      initContainers:
+      - args: 
+        - chown -R 65532:65532 /var/etcd/data
+        command:
+        - sh
+        - -c
+        - --
+        name: change-permissions
+        image: europe-docker.pkg.dev/gardener-project/public/3rd/alpine:3.18.4
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - name: {{ .Release.Name }}-etcd
+          mountPath: /var/etcd/data/
+        securityContext:
+          runAsGroup: 0
+          runAsNonRoot: false
+          runAsUser: 0
       containers:
-      - name: etcd
+      - args: 
+        - start-etcd
+        - --backup-restore-host-port=localhost:{{ .Values.servicePorts.backupRestore }} # {{ .Release.Name }}-etcd-local:{{ .Values.servicePorts.backupRestore }}
+        # - --etcd-server-name={{ .Release.Name }}-etcd-local
+        # - --backup-restore-tls-enabled={{ .Values.backupRestoreTLS }}
+        name: etcd
         image: {{ .Values.images.etcd.repository }}:{{ .Values.images.etcd.tag }}
         imagePullPolicy: {{ .Values.images.etcd.pullPolicy }}
-        command:
-        - /var/etcd/bin/bootstrap.sh
         readinessProbe:
           httpGet:
 {{- if .Values.backupRestoreTLS }}
             scheme: HTTPS
 {{- end }}
             path: /healthz
             port: {{ .Values.servicePorts.backupRestore }}
-          initialDelaySeconds: 5
-          periodSeconds: 5
-        livenessProbe:
-          exec:
-            command:
-            - /bin/sh
-            - -ec
-            - ETCDCTL_API=3
-            - etcdctl
-{{ if .Values.etcdTLS }}
-            - --cert=/var/etcd/ssl/tls/tls.crt
-            - --key=/var/etcd/ssl/tls/tls.key
-            - --cacert=/var/etcd/ssl/ca/ca.crt
-{{ end }}
-            - --endpoints={{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://{{ .Release.Name }}-etcd-0:{{ .Values.servicePorts.client }}
-{{- if and .Values.etcdAuth.username .Values.etcdAuth.password }}
-            - --user={{ .Values.etcdAuth.username }}:{{ .Values.etcdAuth.password }}
-{{- end }}
-            - get
-            - foo
           initialDelaySeconds: 15
           periodSeconds: 5
+          failureThreshold: 5
         ports:
         - containerPort: {{ .Values.servicePorts.server }}
           name: server
@@ -94,8 +95,7 @@ spec:
           mountPath: /var/etcdbr/ssl/ca
 {{- end }}
       - name: backup-restore
-        command:
-        - etcdbrctl
+        args:
         - server
         - --schedule={{ .Values.backup.schedule }}
 {{- if eq .Values.backup.garbageCollectionPolicy "LimitBased" }}
@@ -104,6 +104,7 @@ spec:
 {{- end }}
         - --garbage-collection-period={{ .Values.backup.garbageCollectionPeriod }}
         - --data-dir=/var/etcd/data/new.etcd
+        - --restoration-temp-snapshots-dir=/var/etcd/data/restoration.tmp
         - --storage-provider={{ .Values.backup.storageProvider }}
         - --store-prefix={{ .Release.Name }}-etcd
 {{- if .Values.backup.etcdQuotaBytes }}
@@ -162,6 +163,10 @@ spec:
         resources:
 {{ toYaml .Values.resources.backup | indent 10 }}
         env:
+        - name: "POD_NAME"
+          value: {{ .Release.Name }}-etcd-0
+        - name: "POD_NAMESPACE"
+          value: {{ .Release.Namespace }}
         - name: STORAGE_CONTAINER
           value: {{ .Values.backup.storageContainer }}
 {{- if eq .Values.backup.storageProvider "S3" }}
@@ -180,6 +185,16 @@ spec:
             secretKeyRef:
               name: {{ .Release.Name }}-etcd-backup
               key: "accessKeyID"
+        - name: "AWS_APPLICATION_CREDENTIALS"
+          value: "/var/etcd-backup"
+  {{- if .Values.backup.s3.endpoint }}
+        - name: "AWS_ENDPOINT_URL_S3"
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-etcd-backup
+              key: "endpoint"
+              optional: true
+  {{- end }}
 {{- else if eq .Values.backup.storageProvider "ABS" }}
         - name: "STORAGE_ACCOUNT"
           valueFrom:
@@ -191,6 +206,8 @@ spec:
             secretKeyRef:
               name: {{ .Release.Name }}-etcd-backup
               key: "storageKey"
+        - name: "AZURE_APPLICATION_CREDENTIALS"
+          value: "/var/etcd-backup"
   {{- if .Values.backup.abs.emulatorEnabled }}
         - name: "AZURE_EMULATOR_ENABLED"
           valueFrom:
@@ -209,7 +226,7 @@ spec:
   {{- end }}
 {{- else if eq .Values.backup.storageProvider "GCS" }}
         - name: "GOOGLE_APPLICATION_CREDENTIALS"
-          value: "/root/.gcp/serviceaccount.json"
+          value: "/var/.gcp/serviceaccount.json"
   {{- if .Values.backup.gcs.storageAPIEndpoint }}
         - name: "GOOGLE_STORAGE_API_ENDPOINT"
           valueFrom:
@@ -325,8 +342,16 @@ spec:
 {{- end }}
 {{- if eq .Values.backup.storageProvider "GCS" }}
         - name: etcd-backup
-          mountPath: "/root/.gcp/"
+          mountPath: "/var/.gcp/"
+{{- else if ne .Values.backup.storageProvider "Local" }}
+        - name: etcd-backup
+          mountPath: "/var/etcd-backup/"
 {{- end }}
+      securityContext:
+        runAsGroup: 65532
+        runAsNonRoot: true
+        runAsUser: 65532
+      shareProcessNamespace: true
       volumes:
       - name: etcd-config-file
         configMap:

diff --git a/chart/etcd-backup-restore/values.yaml b/chart/etcd-backup-restore/values.yaml
@@ -1,23 +1,23 @@
 images:
   # etcd image to use
   etcd:
-    repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcd
-    tag: v3.4.13-bootstrap-1
+    repository: europe-docker.pkg.dev/gardener-project/releases/gardener/etcd-wrapper
+    tag: v0.1.1 #v3.4.13-bootstrap-1
     pullPolicy: IfNotPresent
   # etcd-backup-restore image to use
   etcdBackupRestore:
-    repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcdbrctl
-    tag: v0.12.1
+    repository: anveshreddy18/etcd-backup-restore #europe-docker.pkg.dev/gardener-project/snapshots/gardener/etcdbrctl
+    tag: v0.28.0
     pullPolicy: IfNotPresent
 
 resources:
   etcd:
     limits:
-      cpu: 100m
+      cpu: 500m
       memory: 1Gi
     requests:
       cpu: 100m
-      memory: 128Mi
+      memory: 200Mi
   backup:
     limits:
       cpu: 100m
@@ -88,6 +88,7 @@ backup:
   #   region: region-where-bucket-exists
   #   secretAccessKey: secret-access-key-with-object-storage-privileges
   #   accessKeyID: access-key-id-with-route53-privileges
+  #   endpoint: endpoint-override-for-s3 # optional
   #   sseCustomerKey: aes-256-sse-customer-key # optional
   #   sseCustomerAlgorithm: aes-256-sse-customer-algorithm # optional
   # gcs: