Enable etcd-druid to make use of Azurite - the Azure Blob Storage Emulator

Makefile

@@ -52,7 +52,7 @@ revendor: set-permissions
 	@make set-permissions
 
 
-kind-up kind-down ci-e2e-kind deploy-localstack test-e2e: export KUBECONFIG = $(KUBECONFIG_PATH)
+kind-up kind-down ci-e2e-kind ci-e2e-kind-azure deploy-localstack deploy-azurite test-e2e: export KUBECONFIG = $(KUBECONFIG_PATH)
 
 all: druid
 
@@ -169,6 +169,14 @@ kind-down: $(KIND)
 deploy-localstack: $(KUBECTL)
 	./hack/deploy-localstack.sh
 
+.PHONY: deploy-azurite
+deploy-azurite: $(KUBECTL)
+	./hack/deploy-azurite.sh
+
 .PHONY: ci-e2e-kind
 ci-e2e-kind:
 	BUCKET_NAME=$(BUCKET_NAME) ./hack/ci-e2e-kind.sh
+
+.PHONY: ci-e2e-kind-azure
+ci-e2e-kind-azure:
+	BUCKET_NAME=$(BUCKET_NAME) ./hack/ci-e2e-kind-azure.sh

config/samples/druid_v1alpha1_etcd_azurite.yaml

@@ -0,0 +1,77 @@
+apiVersion: druid.gardener.cloud/v1alpha1
+kind: Etcd
+metadata:
+  name: etcd-test
+  labels:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    role: test
+spec:
+  selector:
+    matchLabels:
+      app: etcd-statefulset
+      gardener.cloud/role: controlplane
+      role: test
+  annotations:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    # networking.gardener.cloud/to-dns: allowed
+    # networking.gardener.cloud/to-private-networks: allowed
+    # networking.gardener.cloud/to-public-networks: allowed
+    role: test
+  labels:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    # networking.gardener.cloud/to-dns: allowed
+    # networking.gardener.cloud/to-private-networks: allowed
+    # networking.gardener.cloud/to-public-networks: allowed
+    role: test
+  etcd:
+    metrics: basic
+    defragmentationSchedule: "0 */24 * * *"
+    resources:
+      limits: { cpu: 500m, memory: 1Gi }
+      requests: { cpu: 100m, memory: 200Mi }
+    clientPort: 2379
+    serverPort: 2380
+    quota: 8Gi
+    # heartbeatDuration: 10s
+  backup:
+    port: 8080
+    fullSnapshotSchedule: "0 */24 * * *"
+    resources:
+      limits: { cpu: 200m, memory: 1Gi }
+      requests: { cpu: 23m, memory: 128Mi }
+    garbageCollectionPolicy: Exponential
+    garbageCollectionPeriod: 43200s
+    deltaSnapshotPeriod: 300s
+    deltaSnapshotMemoryLimit: 1Gi
+    store:
+      container: etcd-bucket
+      prefix: etcd-test
+      provider: ABS
+      secretRef:
+        name: etcd-backup-azurite
+    compression:
+      enabled: false
+      policy: "gzip"
+    leaderElection:
+      reelectionPeriod: 5s
+      etcdConnectionTimeout: 5s
+
+  sharedConfig:
+    autoCompactionMode: periodic
+    autoCompactionRetention: "30m"
+  # schedulingConstraints:
+    # affinity: {}
+    # topologySpreadConstraints:
+    # - maxSkew: 1
+    #   topologyKey: topology.kubernetes.io/zone
+    #   whenUnsatisfiable: DoNotSchedule
+    #   labelSelector:
+    #     matchLabels:
+    #       app: etcd-statefulset
+  replicas: 3
+  # priorityClassName: priority-class-name
+  # storageClass: default
+  # storageCapacity: 10Gi

config/samples/etcd-secret-azurite.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    garden.sapcloud.io/role: controlplane
+    role: main
+  name: etcd-backup-azurite
+type: Opaque
+data: 
+  storageAccount: ZGV2c3RvcmVhY2NvdW50MQ==
+  storageKey: RWJ5OHZkTTAyeE5PY3FGbHFVd0pQTGxtRXRsQ0RYSjFPVXpGVDUwdVNSWjZJRnN1RnEyVVZFckN6NEk2dHEvSzFTWkZQVE90ci9LQkhCZWtzb0dNR3c9PQ==
+  emulatorEnabled: dHJ1ZQ== # true
+  storageAPIEndpoint: aHR0cDovL2F6dXJpdGUtc2VydmljZToxMDAwMA== # http://azurite-service:10000, emulatorEnabled has to be true

docs/development/getting-started-locally-azurite.md

@@ -0,0 +1,81 @@
+# Getting started with `etcd-druid` using `Azurite`, and `kind`
+
+This document is a step-by-step guide to run `etcd-druid` with [`Azurite`](https://github.com/Azure/Azurite#introduction), the `Azure Blob Storage` emulator, within a [`kind`](https://kind.sigs.k8s.io/) cluster. This setup is ideal for local development and testing.
+
+## Prerequisites
+
+- [`Docker`](https://www.docker.com/products/docker-desktop/) with the daemon running, or Docker Desktop running.
+- [`Azure CLI`](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) (`>=2.55.0`)
+
+## Environment setup
+
+### Step 1: Provisioning the `kind` cluster
+
+Execute the command below to provision a `kind` cluster. This command also forwards port `10000` from the [`kind`](../../hack/e2e-test/infrastructure/kind/cluster.yaml) cluster to your local machine, enabling `Azurite` access:
+
+``` bash
+make kind-up
+```
+
+Export the `KUBECONFIG` file after running the above command.
+
+### Step 2: Deploy `Azurite`
+
+To start up the `Azurite` emulator in a pod in the `kind` cluster, run:
+
+``` bash
+make deploy-azurite
+```
+
+### Step 3: Set up a `ABS Container`
+
+1. To use the `Azure CLI` with the `Azurite` emulator running as a pod in the `kind` cluster, export the connection string for the `Azure CLI`.
+
+ ``` bash
+ export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+ ```
+
+1. Create a `Azure Blob Storage Container` in `Azurite`
+
+``` bash
+az storage container create -n etcd-bucket
+```
+
+### Step 4: Deploy `etcd-druid`
+
+``` bash
+make deploy
+```
+
+### Step 5: Configure the `Secret` and the `Etcd` manifests
+
+1. Apply the Kubernetes `Secret` manifest through:
+
+``` bash
+kubectl apply -f config/samples/etcd-secret-azurite.yaml
+```
+
+1. Apply the `Etcd` manifest through:
+
+``` bash
+kubectl apply -f config/samples/druid_v1alpha1_etcd_azurite.yaml
+```
+
+### Step 6: Reconcile the `Etcd`
+
+Initiate the `Etcd` reconciliation through the following command:
+
+```bash
+kubectl annotate etcd etcd-test gardener.cloud/operation="reconcile"
+```
+
+### Step 7 : Make use of the Azurite emulator however you wish
+
+`etcd-backup-restore` will now use `Azurite` running in `kind` as the remote store to store snapshots.
+
+### Cleanup
+
+```bash
+make kind-down
+unset AZURE_STORAGE_CONNECTION_STRING KUBECONFIG
+```

hack/ci-e2e-kind-azure.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Constants for Azurite credentials and configurations
+STORAGE_ACCOUNT="devstoreaccount1"
+STORAGE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+AZURITE_ENDPOINT="http://localhost:10000"
+AZURITE_HOST="azurite-service.default:10000"
+AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=${STORAGE_ACCOUNT};AccountKey=${STORAGE_KEY};BlobEndpoint=http://${AZURITE_HOST}/${STORAGE_ACCOUNT};"
+
+make kind-up
+
+trap "
+  ( make kind-down )
+" EXIT
+
+kubectl wait --for=condition=ready node --all
+
+# Setup Azure application credentials
+export AZURE_APPLICATION_CREDENTIALS="/tmp/azuriteCredentials"
+mkdir -p "${AZURE_APPLICATION_CREDENTIALS}"
+echo -n "${STORAGE_ACCOUNT}" > "${AZURE_APPLICATION_CREDENTIALS}/storageAccount"
+echo -n "${STORAGE_KEY}" > "${AZURE_APPLICATION_CREDENTIALS}/storageKey"
+
+# Deploy Azurite and run end-to-end tests
+make deploy-azurite
+make STORAGE_ACCOUNT="${STORAGE_ACCOUNT}" \
+  STORAGE_KEY="${STORAGE_KEY}" \
+  AZURE_STORAGE_API_ENDPOINT="${AZURITE_ENDPOINT}" \
+  EMULATOR_ENABLED="true" \
+  AZURITE_HOST="${AZURITE_HOST}" \
+  AZURE_STORAGE_CONNECTION_STRING="${AZURE_STORAGE_CONNECTION_STRING}" \
+  PROVIDERS="azure" \
+  TEST_ID="$BUCKET_NAME" \
+  STEPS="setup,deploy,test" \
+  test-e2e

hack/deploy-azurite.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -o errexit
+set -o nounset
+set -o pipefail
+
+kubectl apply -f ./hack/e2e-test/infrastructure/azurite/azurite.yaml
+kubectl rollout status deploy/azurite
+kubectl wait --for=condition=ready pod -l app=azurite --timeout=240s

hack/e2e-test/infrastructure/azurite/azurite.yaml

@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: azurite
+  name: azurite
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: azurite
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: azurite
+    spec:
+      containers:
+      - name: azurite
+        image: mcr.microsoft.com/azure-storage/azurite:latest
+        ports:
+          - containerPort: 10000
+            hostPort: 10000
+        command: ["azurite-blob", "--blobHost", "0.0.0.0", "--blobPort", "10000", "--disableProductStyleUrl"]
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: azurite
+  name: azurite-service
+spec:
+  ports:
+  - name: storage-azurite-blobs
+    port: 10000
+    targetPort: 10000 
+  selector:
+    app: azurite

hack/e2e-test/infrastructure/kind/cluster.yaml

@@ -12,4 +12,12 @@ nodes:
     listenAddress: "127.0.0.1"
     # optional: set the protocol to one of TCP, UDP, SCTP.
     # TCP is the default
-    protocol: TCP
+    protocol: TCP
+  - containerPort: 10000
+    hostPort: 10000
+    # optional: set the bind address on the host
+    # 0.0.0.0 is the current default
+    listenAddress: "127.0.0.1"
+    # optional: set the protocol to one of TCP, UDP, SCTP.
+    # TCP is the default
+    protocol: TCP

hack/e2e-test/infrastructure/overlays/azure/common/azure.env

@@ -1,3 +1,4 @@
 STORAGE_ACCOUNT
 STORAGE_KEY
+AZURE_STORAGE_CONNECTION_STRING
 TEST_ID

hack/e2e-test/infrastructure/overlays/azure/common/files/common.sh

@@ -27,7 +27,11 @@ function setup_azcli() {
 
 function create_azure_bucket() {
   echo "Creating ABS bucket ${TEST_ID} in storage account ${STORAGE_ACCOUNT} ..."
-  az storage container create --account-name "${STORAGE_ACCOUNT}" --account-key "${STORAGE_KEY}" --name "${TEST_ID}"
+  if [[ -z "${AZURE_STORAGE_CONNECTION_STRING}" ]]; then
+    az storage container create --account-name "${STORAGE_ACCOUNT}" --account-key "${STORAGE_KEY}" --name "${TEST_ID}"
+  else
+    az storage container create --connection-string "${AZURE_STORAGE_CONNECTION_STRING}" --name "${TEST_ID}"
+  fi
   echo "Successfully created ABS bucket ${TEST_ID} in storage account ${STORAGE_ACCOUNT} ."
 }
 

pkg/common/constants.go

@@ -43,10 +43,15 @@ const (
 	// EnvSourceStorageContainer is the environment variable key for the source storage container.
 	EnvSourceStorageContainer = "SOURCE_STORAGE_CONTAINER"
 
+	// EnvEmulatorEnabled is the environment variable key that is checked to see if storage emulators like Azurite, fake-gcs-emulator are enabled.
+	EnvEmulatorEnabled = "EMULATOR_ENABLED"
+
 	// EnvAWSApplicationCredentials is the environment variable key for AWS application credentials.
 	EnvAWSApplicationCredentials = "AWS_APPLICATION_CREDENTIALS"
 	// EnvAzureApplicationCredentials is the environment variable key for Azure application credentials.
 	EnvAzureApplicationCredentials = "AZURE_APPLICATION_CREDENTIALS"
+	// EnvAzureStorageAPIEndpoint is the environment variable key for Azure storage API endpoint override.
+	EnvAzureStorageAPIEndpoint = "AZURE_STORAGE_API_ENDPOINT"
 	// EnvGoogleApplicationCredentials is the environment variable key for Google application credentials.
 	EnvGoogleApplicationCredentials = "GOOGLE_APPLICATION_CREDENTIALS"
 	// EnvGoogleStorageAPIEndpoint is the environment variable key for Google storage API endpoint override.

pkg/utils/envvar.go

@@ -62,6 +62,8 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error)
 
 	case ABS:
 		envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, credentialsMountPath))
+		envVars = append(envVars, GetEnvVarFromSecret(common.EnvEmulatorEnabled, store.SecretRef.Name, "emulatorEnabled", true))
+		envVars = append(envVars, GetEnvVarFromSecret(common.EnvAzureStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true))
 
 	case GCS:
 		envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, "/var/.gcp/serviceaccount.json"))

test/e2e/utils.go

@@ -381,6 +381,11 @@ func getProviders() ([]TestProvider, error) {
 						},
 					},
 				}
+				azuriteHost := getEnvOrFallback("AZURITE_HOST", "")
+				if azuriteHost != "" {
+					provider.Storage.SecretData["emulatorEnabled"] = []byte("true")
+					provider.Storage.SecretData["storageAPIEndpoint"] = []byte("http://" + azuriteHost)
+				}
 			}
 		case providerGCP:
 			gcsServiceAccountPath := getEnvOrFallback(envGCSServiceAccount, "")