progressivesync_controller_test.go

/*
Copyright 2021 Skyscanner Limited.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"testing"

	syncv1alpha1 "github.com/Skyscanner/applicationset-progressive-sync/api/v1alpha1"
	"github.com/Skyscanner/applicationset-progressive-sync/internal/consts"
	argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
	"github.com/argoproj/gitops-engine/pkg/health"
	"github.com/fluxcd/pkg/apis/meta"
	. "github.com/onsi/gomega"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestRequestsForApplicationChange(t *testing.T) {
	g := NewWithT(t)
	ns, err := createNamespace("progressivesync-test-" + randStringNumber(5))
	g.Expect(err).NotTo(HaveOccurred(), "unable to create namespace")
	defer func() {
		g.Expect(deleteNamespace(ns)).To(Succeed())
	}()

	// Create an ApplicationSet generating Applications
	appSet := "test-requests-app-appset"
	_, err = createApplicationSet(appSet, ns.Name)
	g.Expect(err).NotTo(HaveOccurred())

	// Create a new ProgressiveSync
	ps := newProgressiveSync("app-ownership", ns.Name, appSet)
	g.Expect(k8sClient.Create(ctx, &ps)).To(Succeed())

	// Ensure the reconciliation is successful
	assertHaveCondition(g, ps, meta.ReadyCondition)

	t.Run("filters out events for non-owned applications", func(t *testing.T) {
		// Create an Application which is not generated by the ApplicationSet
		app, err := createApplication("non-owned-app", ns.Name, "wrong-owner")
		g.Expect(err).NotTo(HaveOccurred(), "unable to create applications")

		g.Eventually(func() int {
			requests := reconciler.requestsForApplicationChange(&app)
			return len(requests)
		}).Should(Equal(0))
	})
}

func TestRequestsForSecretChange(t *testing.T) {
	g := NewWithT(t)
	ns, err := createNamespace("progressivesync-test-" + randStringNumber(5))
	g.Expect(err).NotTo(HaveOccurred(), "unable to create namespace")
	defer func() {
		g.Expect(deleteNamespace(ns)).To(Succeed())
	}()

	// Create an ApplicationSet generating Applications
	appSet := "test-requests-secret-appset"
	_, err = createApplicationSet(appSet, ns.Name)
	g.Expect(err).NotTo(HaveOccurred())

	// Create a new ProgressiveSync
	ps := newProgressiveSync("secret-ownership", ns.Name, appSet)
	g.Expect(k8sClient.Create(ctx, &ps)).To(Succeed())

	// Create an Application targeting a specific cluster
	_, err = createApplication("myservice-account-1-eu-west-1a-1", ns.Name, appSet)
	g.Expect(err).NotTo(HaveOccurred())

	// Ensure the reconciliation is successful
	assertHaveCondition(g, ps, meta.ReadyCondition)

	t.Run("send events on a cluster secret change", func(t *testing.T) {
		// Create a cluster secret targeting an existing Application
		secret, err := createSecret("account-1-eu-west-1a-1", ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		g.Eventually(func() int {
			requests := reconciler.requestsForSecretChange(&secret)
			return len(requests)
		}).Should(Equal(1))
	})

	t.Run("filter out events for a generic secret", func(t *testing.T) {
		// Create a secret that is not an ArgoCD cluster
		secret := corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "non-labeled-secret",
				Namespace: ns.Name,
			},
			Data: map[string][]byte{
				"server": []byte("https://cluster.kubernetes.io"),
			},
		}
		g.Expect(k8sClient.Create(ctx, &secret)).To(Succeed())

		g.Eventually(func() int {
			requests := reconciler.requestsForSecretChange(&secret)
			return len(requests)
		}).Should(Equal(0))
	})

	t.Run("filter out events for a cluster secret not targeted by any applications", func(t *testing.T) {
		// Create a secret that is an ArgoCD cluster but without an Application targeting it
		secret := corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "non-matching-secret",
				Namespace: ns.Name,
				Labels: map[string]string{
					consts.ArgoCDSecretTypeLabel: consts.ArgoCDSecretTypeCluster,
				}},
			Data: map[string][]byte{
				"server": []byte("https://cluster.kubernetes.io"),
			},
		}
		g.Expect(k8sClient.Create(ctx, &secret)).To(Succeed())

		g.Eventually(func() int {
			requests := reconciler.requestsForSecretChange(&secret)
			return len(requests)
		}).Should(Equal(0))
	})
}

func TestReconcile(t *testing.T) {
	g := NewWithT(t)
	ns, err := createNamespace("progressivesync-test-" + randStringNumber(5))
	g.Expect(err).NotTo(HaveOccurred(), "unable to create namespace")
	defer func() {
		g.Expect(deleteNamespace(ns)).To(Succeed())
	}()

	t.Run("multi stage", func(t *testing.T) {
		defer mockedClient.Reset()

		// Create an ApplicationSet which generated the Applications.
		appSet := "test-reconcile-appset"
		_, err = createApplicationSet(appSet, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create eight ArgoCD clusters across multiple regions
		secrets := []string{
			"account1-eu-west-1a-1",
			"account1-eu-west-1b-1",
			"account2-eu-central-1a-1",
			"account2-eu-central-1b-1",
			"account3-ap-northeast-1a-1",
			"account3-ap-northeast-1b-1",
			"account4-ap-southeast-1a-1",
			"account4-ap-southeast-1b-1",
		}
		_, err = createSecrets(secrets, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create eight Applications targeting the clusters
		apps := []string{
			"myservice-account1-eu-west-1a-1",
			"myservice-account1-eu-west-1b-1",
			"myservice-account2-eu-central-1a-1",
			"myservice-account2-eu-central-1b-1",
			"myservice-account3-ap-northeast-1a-1",
			"myservice-account3-ap-northeast-1b-1",
			"myservice-account4-ap-southeast-1a-1",
			"myservice-account4-ap-southeast-1b-1",
		}
		_, err = createApplications(apps, ns.Name, appSet)
		g.Expect(err).NotTo(HaveOccurred())

		// Create a multi-stage progressive sync
		ps := newProgressiveSync("multi-stage-ps", ns.Name, appSet)
		ps.Spec.Stages = []syncv1alpha1.Stage{
			newStage("one cluster as canary in eu-west-1", 1, 1, metav1.LabelSelector{
				MatchLabels: map[string]string{
					"region": "eu-west-1",
				}}),
			newStage("one cluster as canary in every other region", 3, 3, metav1.LabelSelector{
				MatchExpressions: []metav1.LabelSelectorRequirement{{
					Key:      "region",
					Operator: metav1.LabelSelectorOpNotIn,
					Values:   []string{"eu-west-1"},
				}}}),
			newStage("remaining clusters", 4, 2, metav1.LabelSelector{}),
		}
		g.Expect(k8sClient.Create(ctx, &ps)).To(Succeed())

		// Ensure the controller started the reconciliation
		assertHaveLastSyncedStage(g, ps, "one cluster as canary in eu-west-1")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusProgressing)

		// Ensure the controller synced the correct Application
		assertHaveSyncedApp(g, "myservice-account1-eu-west-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account1-eu-west-1b-1")
		assertHaveNotSyncedApp(g, "myservice-account2-eu-central-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account2-eu-central-1b-1")
		assertHaveNotSyncedApp(g, "myservice-account3-ap-northeast-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account3-ap-northeast-1b-1")
		assertHaveNotSyncedApp(g, "myservice-account4-ap-southeast-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account4-ap-southeast-1b-1")

		// Set myservice-account1-eu-west-1a-1 as synced
		err = setApplicationSyncStatus("myservice-account1-eu-west-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the controller moves to the second stage once the first is completed
		assertHaveLastSyncedStage(g, ps, "one cluster as canary in every other region")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusProgressing)

		//Ensure the controller synced the correct Applications
		assertHaveSyncedApp(g, "myservice-account2-eu-central-1a-1")
		assertHaveSyncedApp(g, "myservice-account2-eu-central-1b-1")
		assertHaveSyncedApp(g, "myservice-account3-ap-northeast-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account1-eu-west-1b-1")
		assertHaveNotSyncedApp(g, "myservice-account3-ap-northeast-1b-1")
		assertHaveNotSyncedApp(g, "myservice-account4-ap-southeast-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account4-ap-southeast-1b-1")

		// Set the second stage Applications as synced
		err = setApplicationSyncStatus("myservice-account2-eu-central-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account2-eu-central-1b-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account3-ap-northeast-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the controller moved to the third stage once the second stage is completed
		assertHaveLastSyncedStage(g, ps, "remaining clusters")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusProgressing)

		// Ensure the controller synced only the first 2 out of 4 Applications
		assertHaveSyncedApp(g, "myservice-account1-eu-west-1b-1")
		assertHaveSyncedApp(g, "myservice-account3-ap-northeast-1b-1")
		assertHaveNotSyncedApp(g, "myservice-account4-ap-southeast-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account4-ap-southeast-1b-1")

		// Set the first 2 Applications as synced
		err = setApplicationSyncStatus("myservice-account1-eu-west-1b-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account3-ap-northeast-1b-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the controller synced the remaining Applications
		// Using envtually to give time to the controller to pick up the events
		assertHaveSyncedApp(g, "myservice-account4-ap-southeast-1a-1")
		assertHaveSyncedApp(g, "myservice-account4-ap-southeast-1b-1")

		// Set the remaining Applications as synced
		err = setApplicationSyncStatus("myservice-account4-ap-southeast-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account4-ap-southeast-1b-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the reconcile loop is completed
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusCompleted)
		assertHaveCondition(g, ps, meta.ReadyCondition)

		// Ensure the state map is correct
		assertHaveSyncedAtStage(g, ps, "myservice-account1-eu-west-1a-1", "one cluster as canary in eu-west-1")
		assertHaveSyncedAtStage(g, ps, "myservice-account1-eu-west-1b-1", "remaining clusters")
		assertHaveSyncedAtStage(g, ps, "myservice-account2-eu-central-1a-1", "one cluster as canary in every other region")
		assertHaveSyncedAtStage(g, ps, "myservice-account2-eu-central-1b-1", "one cluster as canary in every other region")
		assertHaveSyncedAtStage(g, ps, "myservice-account3-ap-northeast-1a-1", "one cluster as canary in every other region")
		assertHaveSyncedAtStage(g, ps, "myservice-account3-ap-northeast-1b-1", "remaining clusters")
		assertHaveSyncedAtStage(g, ps, "myservice-account4-ap-southeast-1a-1", "remaining clusters")
		assertHaveSyncedAtStage(g, ps, "myservice-account4-ap-southeast-1b-1", "remaining clusters")

		// Ownership test for deletion
		// See https://book.kubebuilder.io/reference/envtest.html#testing-considerations
		boolTrue := true
		expectedOwnerReference := metav1.OwnerReference{
			Kind:               "ProgressiveSync",
			APIVersion:         "argoproj.skyscanner.net/v1alpha1",
			UID:                ps.GetUID(),
			Name:               ps.Name,
			Controller:         &boolTrue,
			BlockOwnerDeletion: &boolTrue,
		}
		g.Eventually(func() []metav1.OwnerReference {
			key := getStateMapNamespacedName(ps)
			cm := corev1.ConfigMap{}
			_ = k8sClient.Get(ctx, key, &cm)
			return cm.ObjectMeta.OwnerReferences
		}).Should(ContainElement(expectedOwnerReference))

	})

	t.Run("complete when no OutOfSync apps", func(t *testing.T) {
		defer mockedClient.Reset()

		// Create an ApplicationSet which generated the Applications.
		appSet := "no-out-of-sync"
		_, err = createApplicationSet(appSet, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create clusters
		secrets := []string{
			"account5-eu-west-1a-1",
			"account5-eu-west-1b-1",
		}
		_, err = createSecrets(secrets, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create Applications targeting the clusters
		apps := []string{
			"myservice-account5-eu-west-1a-1",
			"myservice-account5-eu-west-1b-1",
		}
		_, err = createApplications(apps, ns.Name, appSet)
		g.Expect(err).NotTo(HaveOccurred())

		// Set the Applications as synced
		err = setApplicationSyncStatus("myservice-account5-eu-west-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account5-eu-west-1b-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())

		ps := newProgressiveSync("no-out-of-sync", ns.Name, appSet)
		ps.Spec.Stages = []syncv1alpha1.Stage{
			newStage("stage one", 1, 1, metav1.LabelSelector{}),
		}
		g.Expect(k8sClient.Create(ctx, &ps)).To(Succeed())

		// Ensure the reconciliation is completed
		assertHaveLastSyncedStage(g, ps, "stage one")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusCompleted)
		assertHaveCondition(g, ps, meta.ReadyCondition)

		// Ensure the controller didn't sync any Application
		assertHaveNotSyncedApp(g, "myservice-account5-eu-west-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account5-eu-west-1b-1")

		// Ensure we adopted the synced Application
		assertHaveSyncedAtStage(g, ps, "myservice-account5-eu-west-1a-1", "stage one")
	})

	t.Run("complete when a stage is targeting zero clusters", func(t *testing.T) {
		defer mockedClient.Reset()

		// Create an ApplicationSet which generated the Applications.
		appSet := "zero-clusters"
		_, err = createApplicationSet(appSet, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create a cluster
		_, err = createSecret("account6-eu-west-1a-1", ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create an Application targeting the cluster
		_, err = createApplication("myservice-account6-eu-west-1a-1", ns.Name, appSet)
		g.Expect(err).NotTo(HaveOccurred())

		ps := newProgressiveSync("zero-clusters", ns.Name, appSet)
		ps.Spec.Stages = []syncv1alpha1.Stage{
			newStage("stage-missing-cluster", 1, 1, metav1.LabelSelector{
				MatchLabels: map[string]string{
					"cluster": "missing-cluster",
				}}),
			newStage("stage one cluster", 1, 1, metav1.LabelSelector{
				MatchLabels: map[string]string{
					"cluster": "account6-eu-west-1a-1",
				}}),
			newStage("stage other clusters", 3, 3, metav1.LabelSelector{
				MatchLabels: map[string]string{
					"region": "eu-west-1",
				}}),
		}
		g.Expect(k8sClient.Create(ctx, &ps)).To(Succeed())

		// Set the Application as synced
		err = setApplicationSyncStatus("myservice-account6-eu-west-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the reconciliation is completed
		assertHaveLastSyncedStage(g, ps, "stage other clusters")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusCompleted)

		assertHaveCondition(g, ps, meta.ReadyCondition)
	})

	t.Run("handle failure", func(t *testing.T) {
		defer mockedClient.Reset()

		// Create an ApplicationSet which generated the Applications.
		appSet := "failure"
		_, err = createApplicationSet(appSet, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create clusters
		_, err = createSecret("account7-eu-west-1a-1", ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create Applications targeting the clusters
		_, err = createApplication("myservice-account7-eu-west-1a-1", ns.Name, appSet)
		g.Expect(err).NotTo(HaveOccurred())

		ps := newProgressiveSync("failure", ns.Name, appSet)
		ps.Spec.Stages = []syncv1alpha1.Stage{
			newStage("stage one", 1, 1, metav1.LabelSelector{}),
		}
		g.Expect(k8sClient.Create(ctx, &ps)).To(Succeed())

		// Ensure the reconciliation is progressing
		assertHaveLastSyncedStage(g, ps, "stage one")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusProgressing)

		// Set the Applications as synced but degraded
		err = setApplicationSyncStatus("myservice-account7-eu-west-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationHealthStatus("myservice-account7-eu-west-1a-1", ns.Name, health.HealthStatusDegraded)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the reconciliation is completed
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusFailed)
	})

	t.Run("handle an external process triggering a sync", func(t *testing.T) {
		// Create an ApplicationSet which generated the Applications.
		appSet := "progressing-appset"
		_, err = createApplicationSet(appSet, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create clusters
		secrets := []string{
			"account8-eu-central-1a-1",
			"account8-eu-central-1b-1",
			"account8-eu-west-1a-1",
			"account8-eu-west-1b-1",
		}
		_, err = createSecrets(secrets, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create Applications targeting the clusters
		apps := []string{
			"myservice-account8-eu-central-1a-1",
			"myservice-account8-eu-central-1b-1",
			"myservice-account8-eu-west-1a-1",
			"myservice-account8-eu-west-1b-1",
		}
		_, err = createApplications(apps, ns.Name, appSet)
		g.Expect(err).NotTo(HaveOccurred())

		// Set the Applications as synced and progressing
		err = setApplicationSyncStatus("myservice-account8-eu-west-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationHealthStatus("myservice-account8-eu-west-1a-1", ns.Name, health.HealthStatusProgressing)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account8-eu-west-1b-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationHealthStatus("myservice-account8-eu-west-1b-1", ns.Name, health.HealthStatusProgressing)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account8-eu-central-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationHealthStatus("myservice-account8-eu-central-1a-1", ns.Name, health.HealthStatusProgressing)
		g.Expect(err).NotTo(HaveOccurred())

		ps := newProgressiveSync("progressing", ns.Name, appSet)
		ps.Spec.Stages = []syncv1alpha1.Stage{
			newStage("stage one", 1, 1, metav1.LabelSelector{
				MatchLabels: map[string]string{
					"region": "eu-west-1",
				},
			}),
			newStage("stage two", 2, 2, metav1.LabelSelector{}),
			newStage("stage three", 3, 2, metav1.LabelSelector{}),
		}
		g.Expect(k8sClient.Create(ctx, &ps)).To(Succeed())

		// Ensure the controller didn't sync the selected Application
		assertHaveNotSyncedApp(g, "myservice-account8-eu-central-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account8-eu-central-1b-1")
		assertHaveNotSyncedApp(g, "myservice-account8-eu-west-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account8-eu-west-1b-1")

		// Ensure the controller started the reconciliation
		assertHaveLastSyncedStage(g, ps, "stage one")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusProgressing)

		// Ensure the Applications selected by the stage label selector are synced
		err = setApplicationHealthStatus("myservice-account8-eu-west-1a-1", ns.Name, health.HealthStatusHealthy)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationHealthStatus("myservice-account8-eu-west-1b-1", ns.Name, health.HealthStatusHealthy)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the controller moves to the next stage
		assertHaveLastSyncedStage(g, ps, "stage two")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusProgressing)

		// Ensure the controller synced the correct Applications
		assertHaveNotSyncedApp(g, "myservice-account8-eu-central-1a-1")
		assertHaveSyncedApp(g, "myservice-account8-eu-central-1b-1")

		// Set the Applications healthy
		err = setApplicationHealthStatus("myservice-account8-eu-central-1a-1", ns.Name, health.HealthStatusHealthy)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account8-eu-central-1b-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the reconciliation is completed
		assertHaveLastSyncedStage(g, ps, "stage three")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusCompleted)
		assertHaveCondition(g, ps, meta.ReadyCondition)

		// Ensure the state map is correct
		assertHaveSyncedAtStage(g, ps, "myservice-account8-eu-central-1a-1", "stage two")
		assertHaveSyncedAtStage(g, ps, "myservice-account8-eu-central-1b-1", "stage two")
		assertHaveSyncedAtStage(g, ps, "myservice-account8-eu-west-1a-1", "stage one")
		assertHaveSyncedAtStage(g, ps, "myservice-account8-eu-west-1b-1", "stage one")
	})

	t.Run("handle different revisions", func(t *testing.T) {
		// Create an ApplicationSet which generated the Applications
		appSet := "revision-appset"
		_, err = createApplicationSet(appSet, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create the clusters
		secrets := []string{
			"account9-eu-central-1a-1",
			"account9-eu-central-1b-1",
		}
		_, err = createSecrets(secrets, ns.Name)
		g.Expect(err).NotTo(HaveOccurred())

		// Create the Applications targeting the clusters
		apps := []string{
			"myservice-account9-eu-central-1a-1",
			"myservice-account9-eu-central-1b-1",
		}
		_, err = createApplications(apps, ns.Name, appSet)
		g.Expect(err).NotTo(HaveOccurred())

		// Change one Application revision
		err = setApplicationRevision("myservice-account9-eu-central-1a-1", ns.Name, "zxcvbnm")
		g.Expect(err).NotTo(HaveOccurred())

		// Create the ProgressiveSync
		ps := newProgressiveSync("revision", ns.Name, appSet)
		ps.Spec.Stages = []syncv1alpha1.Stage{
			newStage("single stage", 2, 2, metav1.LabelSelector{
				MatchLabels: map[string]string{
					"region": "eu-central-1",
				},
			}),
		}
		g.Expect(k8sClient.Create(ctx, &ps)).To(Succeed())

		// Ensure the controller didn't sync the selected Applications
		assertHaveNotSyncedApp(g, "myservice-account9-eu-central-1a-1")
		assertHaveNotSyncedApp(g, "myservice-account9-eu-central-1b-1")

		// Ensure the stage is progressing
		assertHaveLastSyncedStage(g, ps, "single stage")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusProgressing)

		// Fix the Application revision
		err = setApplicationRevision("myservice-account9-eu-central-1a-1", ns.Name, Revision)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the controller sync the selected Applications
		assertHaveSyncedApp(g, "myservice-account9-eu-central-1a-1")
		assertHaveSyncedApp(g, "myservice-account9-eu-central-1b-1")

		// Set the Applications as synced
		err = setApplicationSyncStatus("myservice-account9-eu-central-1a-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())
		err = setApplicationSyncStatus("myservice-account9-eu-central-1b-1", ns.Name, argov1alpha1.SyncStatusCodeSynced)
		g.Expect(err).NotTo(HaveOccurred())

		// Ensure the reconciliation is completed
		assertHaveLastSyncedStage(g, ps, "single stage")
		assertHaveLastSyncedStageStatus(g, ps, syncv1alpha1.StageStatusCompleted)
		assertHaveCondition(g, ps, meta.ReadyCondition)
	})
}