diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index fa1bef6599..96f618b369 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -9,12 +9,14 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/openshift/oadp-operator/api/v1alpha1" . "github.com/openshift/oadp-operator/tests/e2e/lib" utils "github.com/openshift/oadp-operator/tests/e2e/utils" - "sigs.k8s.io/controller-runtime/pkg/client" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type VerificationFunction func(client.Client, string) error +type VerificationFunction func(*DpaCustomResource, string) error var _ = Describe("AWS backup restore tests", func() { @@ -53,18 +55,20 @@ var _ = Describe("AWS backup restore tests", func() { PostRestoreVerify VerificationFunction MaxK8SVersion *K8sVersion MinK8SVersion *K8sVersion + dpaCrOpts []DpaCROption + backupOpts []BackupOpts } - mongoReady := VerificationFunction(func(ocClient client.Client, namespace string) error { - Eventually(IsDCReady(ocClient, "mongo-persistent", "todolist"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + mongoReady := VerificationFunction(func(dpaCR *DpaCustomResource, namespace string) error { + Eventually(IsDCReady(dpaCR.Client, "mongo-persistent", "todolist"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) // err := VerifyBackupRestoreData(artifact_dir, namespace, "restify", "parks-app") // TODO: VERIFY PARKS APP DATA return nil }) - mysqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error { + mysqlReady := VerificationFunction(func(dpaCR *DpaCustomResource, namespace string) error { // This test confirms that SCC restore logic in our plugin is working //Eventually(IsDCReady(ocClient, "mssql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - Eventually(IsDeploymentReady(ocClient, "mysql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - exists, err := DoesSCCExist(ocClient, "mysql-persistent-scc") + Eventually(IsDeploymentReady(dpaCR.Client, "mysql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + exists, err := DoesSCCExist(dpaCR.Client, "mysql-persistent-scc") if err != nil { return err } @@ -86,7 +90,7 @@ var _ = Describe("AWS backup restore tests", func() { Skip(reason) } - err := dpaCR.Build(brCase.BackupRestoreType) + err := dpaCR.Build(brCase.BackupRestoreType, brCase.dpaCrOpts...) Expect(err).NotTo(HaveOccurred()) updateLastInstallingNamespace(dpaCR.Namespace) @@ -127,14 +131,14 @@ var _ = Describe("AWS backup restore tests", func() { // Run optional custom verification log.Printf("Running pre-backup function for case %s", brCase.Name) - err = brCase.PreBackupVerify(dpaCR.Client, brCase.ApplicationNamespace) + err = brCase.PreBackupVerify(dpaCR, brCase.ApplicationNamespace) Expect(err).ToNot(HaveOccurred()) nsRequiresResticDCWorkaround, err := NamespaceRequiresResticDCWorkaround(dpaCR.Client, brCase.ApplicationNamespace) Expect(err).ToNot(HaveOccurred()) // create backup log.Printf("Creating backup %s for case %s", backupName, brCase.Name) - backup, err := CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}) + backup, err := CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}, brCase.backupOpts...) Expect(err).ToNot(HaveOccurred()) // wait for backup to not be running @@ -216,7 +220,7 @@ var _ = Describe("AWS backup restore tests", func() { // Run optional custom verification log.Printf("Running post-restore function for case %s", brCase.Name) - err = brCase.PostRestoreVerify(dpaCR.Client, brCase.ApplicationNamespace) + err = brCase.PostRestoreVerify(dpaCR, brCase.ApplicationNamespace) Expect(err).ToNot(HaveOccurred()) // Test is successful, clean up everything @@ -258,5 +262,42 @@ var _ = Describe("AWS backup restore tests", func() { PreBackupVerify: mysqlReady, PostRestoreVerify: mysqlReady, }, nil), + Entry("MySQL application NoDefaultBackupStorageLocation", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-template.yaml", + ApplicationNamespace: "mysql-persistent", + Name: "mysql-e2e", + BackupRestoreType: RESTIC, + PreBackupVerify: VerificationFunction(func(dpaCR *DpaCustomResource, namespace string) error { + // create BSL + err := CreateBackupStorageLocation(dpaCR.Client, velerov1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: dpaCR.Name + "nobsl-1", + Namespace: dpaCR.Namespace, + }, + Spec: *dpaCR.VeleroBSL(), + }) + if err != nil { + return err + } + return mysqlReady(dpaCR, namespace) + }), + PostRestoreVerify: VerificationFunction(func(dpaCR *DpaCustomResource, namespace string) error { + // delete BSL + err := DeleteBackupStorageLocation(dpaCR.Client, velerov1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: dpaCR.Name + "nobsl-1", + Namespace: dpaCR.Namespace, + }}) + if err != nil { + return err + } + return mysqlReady(dpaCR, namespace) + }), + dpaCrOpts: []DpaCROption{ + WithVeleroConfig(&v1alpha1.VeleroConfig{NoDefaultBackupLocation: true}), + WithBackupImages(false), + }, + backupOpts: []BackupOpts{WithBackupStorageLocation("ts-" + instanceName + "nobsl-1")}, // e2e_sute_test.go: dpaCR.name = "ts-" + instanceName + }, nil), ) }) diff --git a/tests/e2e/lib/backup.go b/tests/e2e/lib/backup.go index cb959a4a5d..0b402a410b 100755 --- a/tests/e2e/lib/backup.go +++ b/tests/e2e/lib/backup.go @@ -11,7 +11,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string) (velero.Backup, error) { +type BackupOpts func(*velero.Backup) error + +func WithBackupStorageLocation(name string) BackupOpts { + return func(backup *velero.Backup) error { + backup.Spec.StorageLocation = name + return nil + } +} + +func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string, backupOpts ...BackupOpts) (velero.Backup, error) { backup := velero.Backup{ ObjectMeta: metav1.ObjectMeta{ @@ -22,6 +31,12 @@ func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupNa IncludedNamespaces: namespaces, }, } + for _, opt := range backupOpts { + err := opt(&backup) + if err != nil { + return velero.Backup{}, err + } + } err := ocClient.Create(context.Background(), &backup) return backup, err } diff --git a/tests/e2e/lib/dpa_helpers.go b/tests/e2e/lib/dpa_helpers.go index 09e9800cef..7c078e270a 100755 --- a/tests/e2e/lib/dpa_helpers.go +++ b/tests/e2e/lib/dpa_helpers.go @@ -53,10 +53,55 @@ type DpaCustomResource struct { Provider string } +type DpaCROption func(*oadpv1alpha1.DataProtectionApplication) error + +func WithConfiguration(configuration *oadpv1alpha1.ApplicationConfig) DpaCROption { + return func(cr *oadpv1alpha1.DataProtectionApplication) error { + cr.Spec.Configuration = configuration + return nil + } +} + +func WithVeleroConfig(config *oadpv1alpha1.VeleroConfig) DpaCROption { + return func(cr *oadpv1alpha1.DataProtectionApplication) error { + cr.Spec.Configuration.Velero = config + return nil + } +} + +func WithResticConfig(config *oadpv1alpha1.ResticConfig) DpaCROption { + return func(cr *oadpv1alpha1.DataProtectionApplication) error { + cr.Spec.Configuration.Restic = config + return nil + } +} + +func WithBackupImages(backupImage bool) DpaCROption { + return func(cr *oadpv1alpha1.DataProtectionApplication) error { + cr.Spec.BackupImages = pointer.Bool(backupImage) + return nil + } +} + var VeleroPrefix = "velero-e2e-" + string(uuid.NewUUID()) var Dpa *oadpv1alpha1.DataProtectionApplication -func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error { +func (v *DpaCustomResource) VeleroBSL() *velero.BackupStorageLocationSpec { + return &velero.BackupStorageLocationSpec{ + Provider: v.CustomResource.Spec.BackupLocations[0].Velero.Provider, + Default: true, + Config: v.CustomResource.Spec.BackupLocations[0].Velero.Config, + Credential: v.CustomResource.Spec.BackupLocations[0].Velero.Credential, + StorageType: velero.StorageType{ + ObjectStorage: &velero.ObjectStorageLocation{ + Bucket: v.CustomResource.Spec.BackupLocations[0].Velero.ObjectStorage.Bucket, + Prefix: VeleroPrefix, + }, + }, + } +} + +func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType, dpaCrOpts ...DpaCROption) error { // Velero Instance creation spec with backupstorage location default to AWS. Would need to parameterize this later on to support multiple plugins. dpaInstance := oadpv1alpha1.DataProtectionApplication{ ObjectMeta: metav1.ObjectMeta{ @@ -75,18 +120,7 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error { SnapshotLocations: v.CustomResource.Spec.SnapshotLocations, BackupLocations: []oadpv1alpha1.BackupLocation{ { - Velero: &velero.BackupStorageLocationSpec{ - Provider: v.CustomResource.Spec.BackupLocations[0].Velero.Provider, - Default: true, - Config: v.CustomResource.Spec.BackupLocations[0].Velero.Config, - Credential: v.CustomResource.Spec.BackupLocations[0].Velero.Credential, - StorageType: velero.StorageType{ - ObjectStorage: &velero.ObjectStorageLocation{ - Bucket: v.CustomResource.Spec.BackupLocations[0].Velero.ObjectStorage.Bucket, - Prefix: VeleroPrefix, - }, - }, - }, + Velero: v.VeleroBSL(), }, }, }, @@ -103,6 +137,12 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error { dpaInstance.Spec.Configuration.Velero.DefaultPlugins = append(dpaInstance.Spec.Configuration.Velero.DefaultPlugins, oadpv1alpha1.DefaultPluginCSI) dpaInstance.Spec.Configuration.Velero.FeatureFlags = append(dpaInstance.Spec.Configuration.Velero.FeatureFlags, "EnableCSI") } + + for _, opt := range dpaCrOpts { + if err := opt(&dpaInstance); err != nil { + return err + } + } v.CustomResource = &dpaInstance return nil } diff --git a/tests/e2e/lib/restic_helpers.go b/tests/e2e/lib/restic_helpers.go index 1c5967f20d..42221bc1e2 100755 --- a/tests/e2e/lib/restic_helpers.go +++ b/tests/e2e/lib/restic_helpers.go @@ -6,6 +6,7 @@ import ( "time" oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -177,3 +178,19 @@ func ResticDaemonSetHasNodeSelector(namespace, key, value string) wait.Condition return false, err } } + +func GetResticDaemonsetList(namespace string) (*appsv1.DaemonSetList, error) { + client, err := setUpClient() + if err != nil { + return nil, err + } + registryListOptions := metav1.ListOptions{ + LabelSelector: "component=velero", + } + // get pods in the oadp-operator-e2e namespace with label selector + deploymentList, err := client.AppsV1().DaemonSets(namespace).List(context.TODO(), registryListOptions) + if err != nil { + return nil, err + } + return deploymentList, nil +} diff --git a/tests/e2e/lib/velero_helpers.go b/tests/e2e/lib/velero_helpers.go index d5960c4014..26f7f16775 100644 --- a/tests/e2e/lib/velero_helpers.go +++ b/tests/e2e/lib/velero_helpers.go @@ -19,6 +19,9 @@ import ( veleroClientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/restic" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -155,3 +158,49 @@ func RestoreErrorLogs(ocClient client.Client, restore velero.Restore) []string { } return logLines } + +func CreateBackupStorageLocation(ocClient client.Client, backupStorageLocation velero.BackupStorageLocation) error { + veleroClient, err := GetVeleroClient() + if err != nil { + return err + } + _, err = veleroClient.VeleroV1().BackupStorageLocations(backupStorageLocation.Namespace).Create(context.TODO(), &backupStorageLocation, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return nil + } + return err + } + return nil +} + +func DeleteBackupStorageLocation(ocClient client.Client, backupStorageLocation velero.BackupStorageLocation) error { + veleroClient, err := GetVeleroClient() + if err != nil { + return err + } + err = veleroClient.VeleroV1().BackupStorageLocations(backupStorageLocation.Namespace).Delete(context.TODO(), backupStorageLocation.Name, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + return nil +} + +func GetVeleroDeploymentList(namespace string) (*appsv1.DeploymentList, error) { + client, err := setUpClient() + if err != nil { + return nil, err + } + registryListOptions := metav1.ListOptions{ + LabelSelector: "component=velero", + } + // get pods in the oadp-operator-e2e namespace with label selector + deploymentList, err := client.AppsV1().Deployments(namespace).List(context.TODO(), registryListOptions) + if err != nil { + return nil, err + } + return deploymentList, nil +} diff --git a/tests/e2e/subscription_suite_test.go b/tests/e2e/subscription_suite_test.go index cd9ac0360e..72c8e80a23 100644 --- a/tests/e2e/subscription_suite_test.go +++ b/tests/e2e/subscription_suite_test.go @@ -2,7 +2,6 @@ package e2e_test import ( "context" - "fmt" "log" "time" @@ -82,22 +81,27 @@ var _ = Describe("Subscription Config Suite Test", func() { } if s.Spec.Config != nil && s.Spec.Config.Env != nil { // get pod env vars - log.Printf("Getting velero pods") - podList, err := GetVeleroPods(namespace) + log.Printf("Getting deployments") + vd, err := GetVeleroDeploymentList(namespace) Expect(err).NotTo(HaveOccurred()) - log.Printf("Getting pods containers env vars") - bl := dpaCR.CustomResource.Spec.BackupLocations[0] - for _, podInfo := range podList.Items { - // we care about pods that have labels control-plane=controller-manager, component=velero, "component": "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry", - if podInfo.Labels["control-plane"] == "controller-manager" || - podInfo.Labels["app.kubernetes.io/name"] == "velero" || - podInfo.Labels["component"] == "oadp-"+fmt.Sprintf("%s-%d", dpaCR.Name, 1)+"-"+bl.Velero.Provider+"-registry" { - log.Printf("Checking env vars are passed to each container in " + podInfo.Name) - for _, container := range podInfo.Spec.Containers { + rd, err := GetRegistryDeploymentList(namespace) + Expect(err).NotTo(HaveOccurred()) + log.Printf("Getting daemonsets") + rds, err := GetResticDaemonsetList(namespace) + Expect(err).NotTo(HaveOccurred()) + for _, env := range s.Spec.Config.Env { + for _, deployment := range append(vd.Items, rd.Items...) { + log.Printf("Checking env vars are passed to deployment " + deployment.Name) + for _, container := range deployment.Spec.Template.Spec.Containers { + log.Printf("Checking env vars are passed to container " + container.Name) + Expect(container.Env).To(ContainElement(env)) + } + } + for _, daemonset := range rds.Items { + log.Printf("Checking env vars are passed to daemonset " + daemonset.Name) + for _, container := range daemonset.Spec.Template.Spec.Containers { log.Printf("Checking env vars are passed to container " + container.Name) - for _, env := range s.Spec.Config.Env { - Expect(container.Env).To(ContainElement(env)) - } + Expect(container.Env).To(ContainElement(env)) } } }