From 951c70f48bc78ca8cac566bc16bf8e85ea22d2a9 Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 5 Aug 2023 10:31:52 -0700 Subject: [PATCH 1/8] Fix: AKS tests not working due to invalid K8s version (#6776) * Fix: AKS tests not working due to invalid K8s version * Fixed AKS SKU Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --------- Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- tests/test-infra/azure-aks.bicep | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test-infra/azure-aks.bicep b/tests/test-infra/azure-aks.bicep index 20fb47ca3e4..401628d0f28 100644 --- a/tests/test-infra/azure-aks.bicep +++ b/tests/test-infra/azure-aks.bicep @@ -43,7 +43,7 @@ param diagStorageResourceId string = '' var osDiskSizeGB = 0 // Version of Kubernetes -var kubernetesVersion = '1.25.5' +var kubernetesVersion = '1.26' resource containerRegistry 'Microsoft.ContainerRegistry/registries@2019-05-01' = { name: '${namePrefix}acr' @@ -82,7 +82,7 @@ var networkProfileLinux = { networkPlugin: 'kubenet' } -resource aks 'Microsoft.ContainerService/managedClusters@2021-07-01' = { +resource aks 'Microsoft.ContainerService/managedClusters@2023-05-01' = { location: location name: '${namePrefix}-aks' properties: { @@ -184,8 +184,8 @@ resource aks 'Microsoft.ContainerService/managedClusters@2021-07-01' = { } tags: {} sku: { - name: 'Basic' - tier: 'Paid' + name: 'Base' + tier: 'Standard' } identity: { type: 'SystemAssigned' From 547d390c722676ae71bf2e50854569fc9cd41ad4 Mon Sep 17 00:00:00 2001 From: Josh van Leeuwen Date: Mon, 28 Aug 2023 11:22:38 -0500 Subject: [PATCH 2/8] Create operator scheme before manager to ensure all types are registered (#6853) Signed-off-by: joshvanl Co-authored-by: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> --- pkg/operator/handlers/dapr_handler.go | 1 - pkg/operator/handlers/dapr_handler_test.go | 1 + pkg/operator/operator.go | 44 +++++++++++++++------- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/pkg/operator/handlers/dapr_handler.go b/pkg/operator/handlers/dapr_handler.go index b33b5bb42bd..f1448d3153e 100644 --- a/pkg/operator/handlers/dapr_handler.go +++ b/pkg/operator/handlers/dapr_handler.go @@ -136,7 +136,6 @@ func (h *DaprHandler) Init(ctx context.Context) error { } if h.argoRolloutServiceReconcilerEnabled { - _ = argov1alpha1.AddToScheme(h.Scheme) err = ctrl.NewControllerManagedBy(h.mgr). For(&argov1alpha1.Rollout{}). Owns(&corev1.Service{}). diff --git a/pkg/operator/handlers/dapr_handler_test.go b/pkg/operator/handlers/dapr_handler_test.go index 3be13fb16cf..c6d1d105a4a 100644 --- a/pkg/operator/handlers/dapr_handler_test.go +++ b/pkg/operator/handlers/dapr_handler_test.go @@ -240,6 +240,7 @@ func TestInit(t *testing.T) { mgr := dapr_testing.NewMockManager() _ = scheme.AddToScheme(mgr.GetScheme()) + _ = argov1alpha1.AddToScheme(mgr.GetScheme()) handler := NewDaprHandlerWithOptions(mgr, &Options{ ArgoRolloutServiceReconcilerEnabled: true, diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 3f15e2bdedd..cff301520c1 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -23,6 +23,7 @@ import ( "strings" "time" + argov1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -86,19 +87,6 @@ type operator struct { client client.Client } -var scheme = runtime.NewScheme() - -func init() { - _ = clientgoscheme.AddToScheme(scheme) - - _ = componentsapi.AddToScheme(scheme) - _ = configurationapi.AddToScheme(scheme) - _ = resiliencyapi.AddToScheme(scheme) - _ = httpendpointsapi.AddToScheme(scheme) - _ = subscriptionsapiV1alpha1.AddToScheme(scheme) - _ = subscriptionsapiV2alpha1.AddToScheme(scheme) -} - // NewOperator returns a new Dapr Operator. func NewOperator(ctx context.Context, opts Options) (Operator, error) { conf, err := ctrl.GetConfig() @@ -106,6 +94,12 @@ func NewOperator(ctx context.Context, opts Options) (Operator, error) { return nil, fmt.Errorf("unable to get controller runtime configuration, err: %s", err) } watchdogPodSelector := getSideCarInjectedNotExistsSelector() + + scheme, err := buildScheme(opts) + if err != nil { + return nil, fmt.Errorf("failed to build operator scheme: %w", err) + } + mgr, err := ctrl.NewManager(conf, ctrl.Options{ Scheme: scheme, Port: 19443, @@ -462,3 +456,27 @@ func (r nonLeaderRunnable) Start(ctx context.Context) error { func (r nonLeaderRunnable) NeedLeaderElection() bool { return false } + +func buildScheme(opts Options) (*runtime.Scheme, error) { + builders := []func(*runtime.Scheme) error{ + clientgoscheme.AddToScheme, + componentsapi.AddToScheme, + configurationapi.AddToScheme, + resiliencyapi.AddToScheme, + httpendpointsapi.AddToScheme, + subscriptionsapiV1alpha1.AddToScheme, + subscriptionsapiV2alpha1.AddToScheme, + } + + if opts.ArgoRolloutServiceReconcilerEnabled { + builders = append(builders, argov1alpha1.AddToScheme) + } + + errs := make([]error, len(builders)) + scheme := runtime.NewScheme() + for i, builder := range builders { + errs[i] = builder(scheme) + } + + return scheme, errors.Join(errs...) +} From 58dbabab35c0d09810aeebe2f30478d4e072ab41 Mon Sep 17 00:00:00 2001 From: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> Date: Tue, 29 Aug 2023 19:37:56 +0530 Subject: [PATCH 3/8] Add release notes for 1.11.3 patch Signed-off-by: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> --- docs/release_notes/v1.11.3.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docs/release_notes/v1.11.3.md diff --git a/docs/release_notes/v1.11.3.md b/docs/release_notes/v1.11.3.md new file mode 100644 index 00000000000..cd3adba59f9 --- /dev/null +++ b/docs/release_notes/v1.11.3.md @@ -0,0 +1,26 @@ +# Dapr 1.11.3 + +This update includes a fix for operator deployment on Kubernetes. + - [Fixed Dapr Operator CrashLoopBackOff when Argo Controller is enabled](#fixed-dapr-operator-crashloopbackoff-when-argo-controller-is-enabled) + +## Fixed Dapr Operator CrashLoopBackOff when Argo Controller is enabled + +### Problem + +When using the Dapr Operator on Kubernetes with Argo Controller enabled, the operator would fail to start with the following error: + +``` +time="2023-07-11T17:22:49.937549597Z" level=fatal msg="error running operator: error running operator: failed to wait for rollout caches to sync: timed out waiting for cache to be synced" instance=dapr-operator-7cd6d96dfd-9wbj7 scop │ │ Stream closed EOF for dapr-system/dapr-operator-7cd6d96dfd-9wbj7 (dapr-operator) +``` + +### Impact + +The Dapr Operator would fail to start when Argo Controller is enabled and a Argo rollout is performend. + +### Root cause + +The Dapr Operator was not creating the full Operator scheme before waiting for the controller manager to start. + +### Solution + +Updated the Dapr Operator to create the full Operator scheme before waiting for the controller manager to start. From 5473fbd55a84fb054a0c56546407c5c6873be4be Mon Sep 17 00:00:00 2001 From: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> Date: Tue, 29 Aug 2023 20:50:13 +0530 Subject: [PATCH 4/8] Pin specific version of Redis based on PR #6765 Signed-off-by: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> --- tests/dapr_tests.mk | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/dapr_tests.mk b/tests/dapr_tests.mk index 780c99c1e0b..8ddda76dfc7 100644 --- a/tests/dapr_tests.mk +++ b/tests/dapr_tests.mk @@ -317,7 +317,7 @@ test-deps: # start all e2e tests test-e2e-all: check-e2e-env test-deps # Note: we can set -p 2 to run two tests apps at a time, because today we do not share state between - # tests. In the future, if we add any tests that modify global state (such as dapr config), we'll + # tests. In the future, if we add any tests that modify global state (such as dapr config), we'll # have to be sure and run them after the main test suite, so as not to alter the state of a running # test # Note2: use env variable DAPR_E2E_TEST to pick one e2e test to run. @@ -441,7 +441,13 @@ delete-test-env-k6: # install redis to the cluster without password setup-test-env-redis: - $(HELM) upgrade --install dapr-redis bitnami/redis --wait --timeout 5m0s --namespace $(DAPR_TEST_NAMESPACE) -f ./tests/config/redis_override.yaml + $(HELM) upgrade \ + --install dapr-redis bitnami/redis \ + --version 17.14.5 \ + --wait \ + --timeout 5m0s \ + --namespace $(DAPR_TEST_NAMESPACE) \ + -f ./tests/config/redis_override.yaml delete-test-env-redis: ${HELM} del dapr-redis --namespace ${DAPR_TEST_NAMESPACE} @@ -456,7 +462,7 @@ setup-test-env-rabbitmq: # install mqtt to the cluster setup-test-env-mqtt: - $(HELM) repo add emqx https://repos.emqx.io/charts + $(HELM) repo add emqx https://repos.emqx.io/charts $(HELM) repo update $(HELM) upgrade --install perf-test-emqx emqx/emqx --namespace $(DAPR_TEST_NAMESPACE) --timeout 10m0s @@ -580,13 +586,13 @@ describe-kind-env: export DAPR_TEST_REGISTRY=$${DAPR_TEST_REGISTRY:-localhost:5000/dapr}\n\ export DAPR_TAG=dev\n\ export DAPR_NAMESPACE=dapr-tests" - + delete-kind: docker stop kind-registry && docker rm kind-registry || echo "Could not delete registry." kind delete cluster --name kind -ifeq ($(OS),Windows_NT) +ifeq ($(OS),Windows_NT) detected_OS := windows else detected_OS := $(shell sh -c 'uname 2>/dev/null || echo Unknown' | tr '[:upper:]' '[:lower:]') From 3fbc3b130431264ad143041dfeaf16b1cdb2cb62 Mon Sep 17 00:00:00 2001 From: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> Date: Tue, 29 Aug 2023 21:09:08 +0530 Subject: [PATCH 5/8] Pin specific version of mqtt, kafka, mongodb, pulsar based on PR #6765 Signed-off-by: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> --- tests/dapr_tests.mk | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/tests/dapr_tests.mk b/tests/dapr_tests.mk index 8ddda76dfc7..256ff38e315 100644 --- a/tests/dapr_tests.mk +++ b/tests/dapr_tests.mk @@ -454,23 +454,42 @@ delete-test-env-redis: # install kafka to the cluster setup-test-env-kafka: - $(HELM) upgrade --install dapr-kafka bitnami/kafka -f ./tests/config/kafka_override.yaml --namespace $(DAPR_TEST_NAMESPACE) --timeout 10m0s + $(HELM) upgrade \ + --install dapr-kafka bitnami/kafka \ + --version 23.0.7 \ + -f ./tests/config/kafka_override.yaml \ + --namespace $(DAPR_TEST_NAMESPACE) \ + --timeout 10m0s # install rabbitmq to the cluster setup-test-env-rabbitmq: - $(HELM) upgrade --install rabbitmq bitnami/rabbitmq --set auth.username='admin' --set auth.password='admin' --namespace $(DAPR_TEST_NAMESPACE) --timeout 10m0s + $(HELM) upgrade \ + --install rabbitmq bitnami/rabbitmq \ + --version 12.0.9 \ + --set auth.username='admin' \ + --set auth.password='admin' \ + --namespace $(DAPR_TEST_NAMESPACE) \ + --timeout 10m0s # install mqtt to the cluster setup-test-env-mqtt: $(HELM) repo add emqx https://repos.emqx.io/charts $(HELM) repo update - $(HELM) upgrade --install perf-test-emqx emqx/emqx --namespace $(DAPR_TEST_NAMESPACE) --timeout 10m0s + $(HELM) upgrade \ + --install perf-test-emqx emqx/emqx \ + --version 5.1.4 \ + --namespace $(DAPR_TEST_NAMESPACE) \ + --timeout 10m0s # install mqtt to the cluster setup-test-env-pulsar: $(HELM) repo add apache https://pulsar.apache.org/charts $(HELM) repo update - $(HELM) upgrade --install perf-test-pulsar apache/pulsar --namespace $(DAPR_TEST_NAMESPACE) --timeout 10m0s + $(HELM) upgrade \ + --install perf-test-pulsar apache/pulsar \ + --version 3.0.0 \ + --namespace $(DAPR_TEST_NAMESPACE) \ + --timeout 10m0s # delete kafka from cluster delete-test-env-kafka: @@ -482,7 +501,13 @@ setup-test-env-mongodb: # delete mongodb from cluster delete-test-env-mongodb: - ${HELM} del dapr-mongodb --namespace ${DAPR_TEST_NAMESPACE} + $(HELM) upgrade \ + --install dapr-mongodb bitnami/mongodb \ + --version 13.16.2 \ + -f ./tests/config/mongodb_override.yaml \ + --namespace $(DAPR_TEST_NAMESPACE) \ + --wait \ + --timeout 5m0s # install zipkin to the cluster setup-test-env-zipkin: From 48c0ab70b8353fbaf177888543b6bada1320bfb9 Mon Sep 17 00:00:00 2001 From: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> Date: Tue, 29 Aug 2023 22:04:59 +0530 Subject: [PATCH 6/8] Pin specific version of mongodb based on PR #6765 Signed-off-by: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> --- tests/dapr_tests.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/dapr_tests.mk b/tests/dapr_tests.mk index 256ff38e315..fcf7305c7bd 100644 --- a/tests/dapr_tests.mk +++ b/tests/dapr_tests.mk @@ -497,10 +497,6 @@ delete-test-env-kafka: # install mongodb to the cluster without password setup-test-env-mongodb: - $(HELM) upgrade --install dapr-mongodb bitnami/mongodb -f ./tests/config/mongodb_override.yaml --namespace $(DAPR_TEST_NAMESPACE) --wait --timeout 5m0s - -# delete mongodb from cluster -delete-test-env-mongodb: $(HELM) upgrade \ --install dapr-mongodb bitnami/mongodb \ --version 13.16.2 \ @@ -509,6 +505,10 @@ delete-test-env-mongodb: --wait \ --timeout 5m0s +# delete mongodb from cluster +delete-test-env-mongodb: + ${HELM} del dapr-mongodb --namespace ${DAPR_TEST_NAMESPACE} + # install zipkin to the cluster setup-test-env-zipkin: $(KUBECTL) apply -f ./tests/config/zipkin.yaml -n $(DAPR_TEST_NAMESPACE) From 30fbf0663030b2fa2b50d97db7e689858e006ffb Mon Sep 17 00:00:00 2001 From: halspang <70976921+halspang@users.noreply.github.com> Date: Thu, 24 Aug 2023 16:49:31 -0700 Subject: [PATCH 7/8] Migrate e2e log retention to storage lifecycle (#6850) Signed-off-by: Hal Spang --- tests/test-infra/azure-aks-diagnostic.bicep | 25 +++++++++++++++++++++ tests/test-infra/azure-aks.bicep | 8 ------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/test-infra/azure-aks-diagnostic.bicep b/tests/test-infra/azure-aks-diagnostic.bicep index 846d0d48e07..09fe1d166b5 100644 --- a/tests/test-infra/azure-aks-diagnostic.bicep +++ b/tests/test-infra/azure-aks-diagnostic.bicep @@ -43,5 +43,30 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2021-08-01' = { } } +resource storageManagementPolicies 'Microsoft.Storage/storageAccounts/managementPolicies@2023-01-01' = { + name: 'blobPolicy' + parent: storageAccount + properties: { + policy: { + rules: [ + { + enabled: true + name: 'Delete blob after 15 days' + type: 'Lifecycle' + definition: { + actions: { + baseBlob: { + delete: { + daysAfterModificationGreaterThan: 15 + } + } + } + } + } + ] + } + } +} + output diagLogAnalyticsWorkspaceResourceId string = logAnalyticsWorkspace.id output diagStorageResourceId string = storageAccount.id diff --git a/tests/test-infra/azure-aks.bicep b/tests/test-infra/azure-aks.bicep index 401628d0f28..a3dfe2b365e 100644 --- a/tests/test-infra/azure-aks.bicep +++ b/tests/test-infra/azure-aks.bicep @@ -252,18 +252,10 @@ resource aksDiagnosticStorage 'Microsoft.Insights/diagnosticSettings@2021-05-01- { category: 'kube-apiserver' enabled: true - retentionPolicy: { - days: 15 - enabled: true - } } { category: 'kube-audit' enabled: true - retentionPolicy: { - days: 15 - enabled: true - } } ] storageAccountId: diagStorageResourceId From 655495703a1cc00e0ab2846af6ab6dc4131a3f00 Mon Sep 17 00:00:00 2001 From: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> Date: Wed, 30 Aug 2023 08:59:01 +0530 Subject: [PATCH 8/8] Add postgres env setup commands in make file Signed-off-by: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com> --- tests/config/postgres_override.yaml | 28 ++++++++++++++++++++++++++++ tests/dapr_tests.mk | 18 ++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 tests/config/postgres_override.yaml diff --git a/tests/config/postgres_override.yaml b/tests/config/postgres_override.yaml new file mode 100644 index 00000000000..bb22a0f0e92 --- /dev/null +++ b/tests/config/postgres_override.yaml @@ -0,0 +1,28 @@ +global: + postgresql: + auth: + username: postgres + postgresPassword: example + database: dapr_test +primary: + initdb: + scripts: + init.sql: | + CREATE TABLE IF NOT EXISTS configtable (KEY VARCHAR NOT NULL, VALUE VARCHAR NOT NULL, VERSION VARCHAR NOT NULL, METADATA JSON); + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + persistence: + enabled: false +tls: + enabled: false \ No newline at end of file diff --git a/tests/dapr_tests.mk b/tests/dapr_tests.mk index fcf7305c7bd..b7cc6a77064 100644 --- a/tests/dapr_tests.mk +++ b/tests/dapr_tests.mk @@ -252,7 +252,7 @@ create-test-namespace: delete-test-namespace: kubectl delete namespace $(DAPR_TEST_NAMESPACE) -setup-3rd-party: setup-helm-init setup-test-env-redis setup-test-env-kafka setup-test-env-mongodb setup-test-env-zipkin +setup-3rd-party: setup-helm-init setup-test-env-redis setup-test-env-kafka setup-test-env-mongodb setup-test-env-zipkin setup-test-env-postgres setup-pubsub-subs-perf-test-components: setup-test-env-rabbitmq setup-test-env-pulsar setup-test-env-mqtt @@ -505,6 +505,20 @@ setup-test-env-mongodb: --wait \ --timeout 5m0s +# install postgres to the cluster +setup-test-env-postgres: + $(HELM) upgrade \ + --install dapr-postgres bitnami/postgresql \ + --version 12.8.0 \ + -f ./tests/config/postgres_override.yaml \ + --namespace $(DAPR_TEST_NAMESPACE) \ + --wait \ + --timeout 5m0s + +# delete postgres from cluster +delete-test-env-postgres: + $(HELM) del dapr-postgres --namespace $(DAPR_TEST_NAMESPACE) + # delete mongodb from cluster delete-test-env-mongodb: ${HELM} del dapr-mongodb --namespace ${DAPR_TEST_NAMESPACE} @@ -516,7 +530,7 @@ delete-test-env-zipkin: $(KUBECTL) delete -f ./tests/config/zipkin.yaml -n $(DAPR_TEST_NAMESPACE) # Setup the test environment by installing components -setup-test-env: setup-test-env-kafka setup-test-env-redis setup-test-env-mongodb setup-test-env-k6 setup-test-env-zipkin +setup-test-env: setup-test-env-kafka setup-test-env-redis setup-test-env-mongodb setup-test-env-postgres setup-test-env-k6 setup-test-env-zipkin save-dapr-control-plane-k8s-resources: mkdir -p '$(DAPR_CONTAINER_LOG_PATH)'