Added support to etcd-druid to make use of Azurite - the Azure Bl…

…ob Storage Emulator

* New documentation `docs/development/getting-started-locally-azurite.md`, to document
  the setup process for running `etcd-druid` with `Azurite`.
* New make target `deploy-azurite` to setup the `Azurite` emulator in the `kind` cluster.
* New make target `ci-e2e-kind-azurite` to run the e2e tests with `Azurite`.
* New script `hack/deploy-azurite.sh`, to setup the `Azurite` emulator in the `kind` cluster.
* New script `ci-e2e-kind-azurite`, to run the e2e tests with `Azurite`.
* New manifest `config/samples/druid_v1alpha1_etcd_azurite.yaml`, to run with `Azurite`.
* New manifest `config/samples/etcd-secret-azurite.yaml`, to run with with `Azurite`.
* New manifest `hack/e2e-test/infrastructure/azurite/azurite.yaml`, to setup `Azurite`.
* Port `10000` forward `hack/e2e-test/infrastructure/kind/cluster.yaml`, to setup `Azurite`.
* New environment variable `AZURE_STORAGE_API_ENDPOINT` to override ABS endpoint to `Azurite`.
* Environment variable `AZURE_STORAGE_CONNECTION_STRING` added to
  `hack/e2e-test/infrastructure/overlays/azure/common/azure.env` to enable the script which
  creates the ABS Container in `Azurite` to access `Azurite` using the relevant Storage Account,
  Storage Key, and the BlobEndpoint at which `Azurite`'s Blob Storage is hosted.
* Modified `hack/e2e-test/infrastructure/overlays/azure/common/files/common.sh` to create
  the ABS Container in `Azurite` when the relevant environment variable is present.
* Modified `pkg/utils/envvar.go` to read the Azure infrastructure `Secret` for the fields
  `enableAzurite` and `storageAPIEndpoint`, which are passed to the `etcd-backup-restore`
  container as environment variables, which are used to make use of the `Azurite` emulator.
* Modified `pkg/common/constants.go` to add `AZURE_ENABLE_STORAGE_EMULATOR` and
  `AZURE_STORAGE_API_ENDPOINT`, which are the enviroment variables passed to
  `etcd-backup-restore` to make use of the `Azurite` emulator.
* Modified `test/e2e/utils.go` to check for the `AZURITE_HOST` environment variable
  which creates the Azure `Secret`, to include additional fields `enableAzurite` and
  `storageAPIEndpoint` to enable usage of `Azurite`.

Fixed integration tests

Makefile

@@ -52,7 +52,7 @@ revendor: set-permissions
 	@make set-permissions
 
 
-kind-up kind-down ci-e2e-kind deploy-localstack test-e2e: export KUBECONFIG = $(KUBECONFIG_PATH)
+kind-up kind-down ci-e2e-kind ci-e2e-kind-azurite deploy-localstack deploy-azurite test-e2e: export KUBECONFIG = $(KUBECONFIG_PATH)
 
 all: druid
 
@@ -169,6 +169,14 @@ kind-down: $(KIND)
 deploy-localstack: $(KUBECTL)
 	./hack/deploy-localstack.sh
 
+.PHONY: deploy-azurite
+deploy-azurite: $(KUBECTL)
+	./hack/deploy-azurite.sh
+
 .PHONY: ci-e2e-kind
 ci-e2e-kind:
 	BUCKET_NAME=$(BUCKET_NAME) ./hack/ci-e2e-kind.sh
+
+.PHONY: ci-e2e-kind-azurite
+ci-e2e-kind-azurite:
+	BUCKET_NAME=$(BUCKET_NAME) ./hack/ci-e2e-kind-azurite.sh

config/samples/druid_v1alpha1_etcd_azurite.yaml

@@ -0,0 +1,77 @@
+apiVersion: druid.gardener.cloud/v1alpha1
+kind: Etcd
+metadata:
+  name: etcd-test
+  labels:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    role: test
+spec:
+  selector:
+    matchLabels:
+      app: etcd-statefulset
+      gardener.cloud/role: controlplane
+      role: test
+  annotations:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    # networking.gardener.cloud/to-dns: allowed
+    # networking.gardener.cloud/to-private-networks: allowed
+    # networking.gardener.cloud/to-public-networks: allowed
+    role: test
+  labels:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    # networking.gardener.cloud/to-dns: allowed
+    # networking.gardener.cloud/to-private-networks: allowed
+    # networking.gardener.cloud/to-public-networks: allowed
+    role: test
+  etcd:
+    metrics: basic
+    defragmentationSchedule: "0 */24 * * *"
+    resources:
+      limits: { cpu: 500m, memory: 1Gi }
+      requests: { cpu: 100m, memory: 200Mi }
+    clientPort: 2379
+    serverPort: 2380
+    quota: 8Gi
+    # heartbeatDuration: 10s
+  backup:
+    port: 8080
+    fullSnapshotSchedule: "0 */24 * * *"
+    resources:
+      limits: { cpu: 200m, memory: 1Gi }
+      requests: { cpu: 23m, memory: 128Mi }
+    garbageCollectionPolicy: Exponential
+    garbageCollectionPeriod: 43200s
+    deltaSnapshotPeriod: 300s
+    deltaSnapshotMemoryLimit: 1Gi
+    store:
+      container: etcd-bucket
+      prefix: etcd-test
+      provider: ABS
+      secretRef:
+        name: etcd-backup-azurite
+    compression:
+      enabled: false
+      policy: "gzip"
+    leaderElection:
+      reelectionPeriod: 5s
+      etcdConnectionTimeout: 5s
+
+  sharedConfig:
+    autoCompactionMode: periodic
+    autoCompactionRetention: "30m"
+  # schedulingConstraints:
+    # affinity: {}
+    # topologySpreadConstraints:
+    # - maxSkew: 1
+    #   topologyKey: topology.kubernetes.io/zone
+    #   whenUnsatisfiable: DoNotSchedule
+    #   labelSelector:
+    #     matchLabels:
+    #       app: etcd-statefulset
+  replicas: 3
+  # priorityClassName: priority-class-name
+  # storageClass: default
+  # storageCapacity: 10Gi

config/samples/etcd-secret-azurite.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    garden.sapcloud.io/role: controlplane
+    role: main
+  name: etcd-backup-azurite
+type: Opaque
+data: 
+  storageAccount: ZGV2c3RvcmVhY2NvdW50MQ==
+  storageKey: RWJ5OHZkTTAyeE5PY3FGbHFVd0pQTGxtRXRsQ0RYSjFPVXpGVDUwdVNSWjZJRnN1RnEyVVZFckN6NEk2dHEvSzFTWkZQVE90ci9LQkhCZWtzb0dNR3c9PQ==
+  enableAzurite: dHJ1ZQ== # true
+  storageAPIEndpoint: aHR0cDovL2F6dXJpdGUtc2VydmljZToxMDAwMA== # http://azurite-service:10000, enableAzurite has to be true
+  # storageAPIEndpoint: aHR0cDovL2F6dXJpdGUtc2VydmljZS5kZWZhdWx0OjEwMDAw # http://azurite-service.default:10000

docs/development/getting-started-locally-azurite.md

@@ -0,0 +1,82 @@
+# Getting started with `etcd-druid` using `Azurite`, and `kind`
+
+This document is a step-by-step guide to run `etcd-druid` with [`Azurite`](https://github.com/Azure/Azurite#introduction), the `Azure Blob Storage` emulator, within a [`kind`](https://kind.sigs.k8s.io/) cluster. This setup is ideal for local development and testing.
+
+## Prerequisites
+
+- [`Docker`](https://www.docker.com/products/docker-desktop/) with the daemon running, or Docker Desktop running.
+- [`Azure CLI`](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) (`>=2.55.0`)
+
+## Environment setup
+
+### Step 1: Provisioning the `kind` cluster
+
+Execute the command below to provision a `kind` cluster. This command also forwards port `10000` from the [`kind`](../../hack/e2e-test/infrastructure/kind/cluster.yaml) cluster to your local machine, enabling `Azurite` access:
+
+``` bash
+make kind-up
+```
+
+Export the `KUBECONFIG` file after running the above command.
+
+
+### Step 2: Deploy `Azurite`
+
+To start up the `Azurite` emulator in a pod in the `kind` cluster, run:
+
+``` bash
+make deploy-azurite
+```
+
+### Step 3: Set up an `ABS Container`
+
+1. To use the `Azure CLI` with the `Azurite` emulator running as a pod in the `kind` cluster, export the connection string for the `Azure CLI`.
+
+ ``` bash
+ export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+ ```
+
+2. Create an `Azure Blob Storage Container` in `Azurite`
+
+``` bash
+az storage container create -n etcd-bucket
+```
+
+### Step 4: Deploy `etcd-druid`
+
+``` bash
+make deploy
+```
+
+### Step 5: Configure the `Secret` and the `Etcd` manfiests
+
+1. Apply the Kubernetes `Secret` manifest through:
+
+``` bash
+kubectl apply -f config/samples/etcd-secret-azurite.yaml
+```
+
+2. Apply the `Etcd` manifest through:
+
+``` bash
+kubectl apply -f config/samples/druid_v1alpha1_etcd_azurite.yaml
+```
+
+### Step 6: Reconcile the `Etcd`
+
+Initiate the `Etcd` reconciliation through the following command:
+
+```bash
+kubectl annotate etcd etcd-test gardener.cloud/operation="reconcile"
+```
+
+### Step 7 : Make use of the Azurite emulator however you wish
+
+`etcd-backup-restore` will now use `Azurite` running in `kind` as the remote store to store snapshots.
+
+### Cleanup
+
+```bash
+make kind-down
+unset AZURE_STORAGE_CONNECTION_STRING KUBECONFIG
+```

hack/ci-e2e-kind-azurite.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -o errexit
+set -o nounset
+set -o pipefail
+
+make kind-up
+
+trap "
+  ( make kind-down )
+" EXIT
+
+kubectl wait --for=condition=ready node --all
+
+export AZURE_APPLICATION_CREDENTIALS="/tmp/azuriteCredentials"
+
+mkdir -p /tmp/azuriteCredentials/
+echo -n "devstoreaccount1" > /tmp/azuriteCredentials/storageAccount
+echo -n "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" > /tmp/azuriteCredentials/storageKey
+
+make deploy-azurite
+make STORAGE_ACCOUNT="devstoreaccount1" \
+  STORAGE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+  AZURE_STORAGE_API_ENDPOINT="http://localhost:10000" \
+  AZURE_ENABLE_STORAGE_EMULATOR="true" \
+  AZURITE_HOST="azurite-service.default:10000" \
+  AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.default:10000/devstoreaccount1;" \
+  PROVIDERS="azure" \
+  TEST_ID="$BUCKET_NAME" \
+  STEPS="setup,deploy,test" \
+  test-e2e

hack/deploy-azurite.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -o errexit
+set -o nounset
+set -o pipefail
+
+kubectl apply -f ./hack/e2e-test/infrastructure/azurite/azurite.yaml
+kubectl rollout status deploy/azurite
+kubectl wait --for=condition=ready pod -l app=azurite --timeout=240s

hack/e2e-test/infrastructure/azurite/azurite.yaml

@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: azurite
+  name: azurite
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: azurite
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: azurite
+    spec:
+      containers:
+      - name: azurite
+        image: mcr.microsoft.com/azure-storage/azurite:latest
+        ports:
+          - containerPort: 10000
+            hostPort: 10000
+        command: ["azurite-blob", "--blobHost", "0.0.0.0", "--blobPort", "10000", "--disableProductStyleUrl"]
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: azurite
+  name: azurite-service
+spec:
+  ports:
+  - name: storage-azurite-blobs
+    port: 10000
+    targetPort: 10000 
+  selector:
+    app: azurite

hack/e2e-test/infrastructure/kind/cluster.yaml

@@ -12,4 +12,12 @@ nodes:
     listenAddress: "127.0.0.1"
     # optional: set the protocol to one of TCP, UDP, SCTP.
     # TCP is the default
-    protocol: TCP
+    protocol: TCP
+  - containerPort: 10000
+    hostPort: 10000
+    # optional: set the bind address on the host
+    # 0.0.0.0 is the current default
+    listenAddress: "127.0.0.1"
+    # optional: set the protocol to one of TCP, UDP, SCTP.
+    # TCP is the default
+    protocol: TCP

hack/e2e-test/infrastructure/overlays/azure/common/azure.env

@@ -1,3 +1,4 @@
 STORAGE_ACCOUNT
 STORAGE_KEY
+AZURE_STORAGE_CONNECTION_STRING
 TEST_ID

hack/e2e-test/infrastructure/overlays/azure/common/files/common.sh

@@ -27,7 +27,11 @@ function setup_azcli() {
 
 function create_azure_bucket() {
   echo "Creating ABS bucket ${TEST_ID} in storage account ${STORAGE_ACCOUNT} ..."
-  az storage container create --account-name "${STORAGE_ACCOUNT}" --account-key "${STORAGE_KEY}" --name "${TEST_ID}"
+  if [[ -z "${AZURE_STORAGE_CONNECTION_STRING}" ]]; then
+    az storage container create --account-name "${STORAGE_ACCOUNT}" --account-key "${STORAGE_KEY}" --name "${TEST_ID}"
+  else
+    az storage container create --connection-string "${AZURE_STORAGE_CONNECTION_STRING}" --name "${TEST_ID}"
+  fi
   echo "Successfully created ABS bucket ${TEST_ID} in storage account ${STORAGE_ACCOUNT} ."
 }
 

pkg/common/constants.go

@@ -47,6 +47,10 @@ const (
 	EnvAWSApplicationCredentials = "AWS_APPLICATION_CREDENTIALS"
 	// EnvAzureApplicationCredentials is the environment variable key for Azure application credentials.
 	EnvAzureApplicationCredentials = "AZURE_APPLICATION_CREDENTIALS"
+	// EnvAzureEnableAzurite is the environment variable key to enable the ABS emulator Azurite.
+	EnvAzureEnableAzurite = "AZURE_ENABLE_STORAGE_EMULATOR"
+	// EnvAzureStorageAPIEndpoint is the environment variable key for Azure storage API endpoint override.
+	EnvAzureStorageAPIEndpoint = "AZURE_STORAGE_API_ENDPOINT"
 	// EnvGoogleApplicationCredentials is the environment variable key for Google application credentials.
 	EnvGoogleApplicationCredentials = "GOOGLE_APPLICATION_CREDENTIALS"
 	// EnvGoogleStorageAPIEndpoint is the environment variable key for Google storage API endpoint override.

pkg/utils/envvar.go

@@ -62,6 +62,8 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error)
 
 	case ABS:
 		envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, credentialsMountPath))
+		envVars = append(envVars, GetEnvVarFromSecret(common.EnvAzureEnableAzurite, store.SecretRef.Name, "enableAzurite", true))
+		envVars = append(envVars, GetEnvVarFromSecret(common.EnvAzureStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true))
 
 	case GCS:
 		envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, "/var/.gcp/serviceaccount.json"))

test/e2e/utils.go

@@ -381,6 +381,11 @@ func getProviders() ([]TestProvider, error) {
 						},
 					},
 				}
+				azuriteHost := getEnvOrFallback("AZURITE_HOST", "")
+				if azuriteHost != "" {
+					provider.Storage.SecretData["enableAzurite"] = []byte("true")
+					provider.Storage.SecretData["storageAPIEndpoint"] = []byte("http://" + azuriteHost)
+				}
 			}
 		case providerGCP:
 			gcsServiceAccountPath := getEnvOrFallback(envGCSServiceAccount, "")