From 02eb6225704e8e0330d3ff6a8567e40284462892 Mon Sep 17 00:00:00 2001 From: hhpatel14 Date: Mon, 1 Nov 2021 23:35:05 -0400 Subject: [PATCH 1/8] CSI Backrestore E2E test --- tests/e2e/backup_restore_suite_test.go | 81 +++++--- .../mssql-persistent-csi-template.yaml | 177 ++++++++++++++++++ .../mssql-persistent/volumeSnapshotClass.yaml | 14 ++ tests/e2e/velero_helpers.go | 40 ++-- 4 files changed, 275 insertions(+), 37 deletions(-) create mode 100644 tests/e2e/sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml create mode 100644 tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index fe6ace1e67..1b2c3a8ede 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -23,25 +23,6 @@ var _ = Describe("AWS backup restore tests", func() { err = createCredentialsSecret(credData, namespace, credSecretRef) Expect(err).NotTo(HaveOccurred()) - - err = vel.Build() - Expect(err).NotTo(HaveOccurred()) - - err = vel.CreateOrUpdate(&vel.CustomResource.Spec) - Expect(err).NotTo(HaveOccurred()) - - log.Printf("Waiting for velero pod to be running") - Eventually(isVeleroPodRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - - if vel.CustomResource.Spec.EnableRestic == nil || *vel.CustomResource.Spec.EnableRestic { - log.Printf("Waiting for restic pods to be running") - Eventually(areResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - } - - if vel.CustomResource.Spec.BackupImages == nil || *vel.CustomResource.Spec.BackupImages { - log.Printf("Waiting for registry pods to be running") - Eventually(areRegistryDeploymentsAvailable(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - } }) var _ = AfterEach(func() { @@ -56,6 +37,7 @@ var _ = Describe("AWS backup restore tests", func() { ApplicationTemplate string ApplicationNamespace string Name string + BackupRestoreType string PreBackupVerify VerificationFunction PostRestoreVerify VerificationFunction MaxK8SVersion *k8sVersion @@ -64,6 +46,32 @@ var _ = Describe("AWS backup restore tests", func() { DescribeTable("backup and restore applications", func(brCase BackupRestoreCase, expectedErr error) { + + err := vel.Build(brCase.BackupRestoreType) + Expect(err).NotTo(HaveOccurred()) + + err = vel.CreateOrUpdate(&vel.CustomResource.Spec) + Expect(err).NotTo(HaveOccurred()) + + log.Printf("Waiting for velero pod to be running") + Eventually(isVeleroPodRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) + + if brCase.BackupRestoreType == "restic" { + if vel.CustomResource.Spec.EnableRestic == nil || *vel.CustomResource.Spec.EnableRestic { + log.Printf("Waiting for restic pods to be running") + Eventually(areResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) + } + } + if brCase.BackupRestoreType == "csi" { + log.Printf("Creating VolumeSnapshot for CSI backuprestore of %s", brCase.Name) + err = installApplication(vel.Client, "./sample-applications/mssql-persistent/volumeSnapshotClass.yaml") + Expect(err).ToNot(HaveOccurred()) + } + + if vel.CustomResource.Spec.BackupImages == nil || *vel.CustomResource.Spec.BackupImages { + log.Printf("Waiting for registry pods to be running") + Eventually(areRegistryDeploymentsAvailable(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) + } if notVersionTarget, reason := NotServerVersionTarget(brCase.MinK8SVersion, brCase.MaxK8SVersion); notVersionTarget { Skip(reason) } @@ -74,7 +82,7 @@ var _ = Describe("AWS backup restore tests", func() { // install app log.Printf("Installing application for case %s", brCase.Name) - err := installApplication(vel.Client, brCase.ApplicationTemplate) + err = installApplication(vel.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) // wait for pods to be running Eventually(areApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) @@ -131,11 +139,18 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Uninstalling application for case %s", brCase.Name) err = uninstallApplication(vel.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) + + if brCase.BackupRestoreType == "csi" { + log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name) + err = uninstallApplication(vel.Client, "./sample-applications/mssql-persistent/volumeSnapshotClass.yaml") + Expect(err).ToNot(HaveOccurred()) + } }, - Entry("MSSQL application", BackupRestoreCase{ - ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-template.yaml", + Entry("MSSQL application CSI", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml", ApplicationNamespace: "mssql-persistent", Name: "mssql-e2e", + BackupRestoreType: "csi", PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { return nil }), @@ -155,6 +170,7 @@ var _ = Describe("AWS backup restore tests", func() { ApplicationTemplate: "./sample-applications/parks-app/manifest.yaml", ApplicationNamespace: "parks-app", Name: "parks-e2e", + BackupRestoreType: "restic", PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) return nil @@ -164,10 +180,31 @@ var _ = Describe("AWS backup restore tests", func() { }), MaxK8SVersion: &k8sVersionOcp47, }, nil), + Entry("MSSQL application", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-template.yaml", + ApplicationNamespace: "mssql-persistent", + Name: "mssql-e2e", + BackupRestoreType: "restic", + PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { + return nil + }), + PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { + // This test confirms that SCC restore logic in our plugin is working + exists, err := doesSCCExist(ocClient, "mssql-persistent-scc") + if err != nil { + return err + } + if !exists { + return errors.New("did not find MSSQL scc after restore") + } + return nil + }), + }, nil), Entry("Parks application >=4.8.0", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/parks-app/manifest4.8.yaml", ApplicationNamespace: "parks-app", Name: "parks-e2e", + BackupRestoreType: "restic", PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) return nil diff --git a/tests/e2e/sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml b/tests/e2e/sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml new file mode 100644 index 0000000000..0dcd7f394f --- /dev/null +++ b/tests/e2e/sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml @@ -0,0 +1,177 @@ +apiVersion: v1 +kind: List +items: + - kind: Namespace + apiVersion: v1 + metadata: + name: mssql-persistent + labels: + app: mssql + - apiVersion: v1 + kind: Secret + metadata: + name: mssql-secret + namespace: mssql-persistent + labels: + app: mssql + stringData: + mssql-password: P@ssw0rd1! + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: mssql-persistent-sa + namespace: mssql-persistent + labels: + component: mssql-persistent + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: mssql-pvc + namespace: mssql-persistent + labels: + app: mssql + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: gp2-csi + - kind: SecurityContextConstraints + apiVersion: security.openshift.io/v1 + metadata: + name: mssql-persistent-scc + allowPrivilegeEscalation: true + allowPrivilegedContainer: true + runAsUser: + type: RunAsAny + seLinuxContext: + type: RunAsAny + fsGroup: + type: RunAsAny + supplementalGroups: + type: RunAsAny + volumes: + - '*' + users: + - system:admin + - system:serviceaccount:mssql-persistent:mssql-persistent-sa + - apiVersion: apps.openshift.io/v1 + kind: DeploymentConfig + metadata: + name: mssql-deployment + namespace: mssql-persistent + labels: + e2e-app: "true" + app: mssql + spec: + replicas: 1 + selector: + name: mssql + strategy: + type: Recreate + template: + metadata: + labels: + e2e-app: "true" + name: mssql + app: mssql + spec: + serviceAccountName: mssql-persistent-sa + containers: + - env: + - name: ACCEPT_EULA + value: "Y" + - name: SA_PASSWORD + valueFrom: + secretKeyRef: + key: mssql-password + name: mssql-secret + image: quay.io/ocpmigrate/mssql-server:latest + imagePullPolicy: Always + name: mssql + securityContext: + privileged: true + ports: + - containerPort: 1433 + resources: + limits: + memory: "3Gi" + cpu: "0.5" + requests: + memory: "3Gi" + cpu: "0.5" + volumeMounts: + - mountPath: "/var/opt/mssql/data" + name: mssql-vol + volumes: + - name: mssql-vol + persistentVolumeClaim: + claimName: mssql-pvc + - apiVersion: v1 + kind: Service + metadata: + name: mssql-service + namespace: mssql-persistent + spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 + type: ClusterIP + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: mssql-app-deployment + namespace: mssql-persistent + labels: + e2e-app: "true" + spec: + replicas: 1 + selector: + matchLabels: + app: mssql-app + template: + metadata: + labels: + e2e-app: "true" + app: mssql-app + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: mssql-persistent-sa + containers: + - name: mssql-app + image: quay.io/ocpmigrate/mssql-sample-app:microsoft + imagePullPolicy: Always + ports: + - containerPort: 5000 + securityContext: + privileged: true + env: + - name: ConnString + value: "Server=mssql-service.mssql-persistent.svc.cluster.local;Database=ProductCatalog;User ID=WebLogin; password=SQLPass1234!" + - apiVersion: v1 + kind: Service + metadata: + name: mssql-app-service + namespace: mssql-persistent + spec: + selector: + app: mssql-app + ports: + - name: "tcp" + protocol: TCP + port: 5000 + targetPort: 5000 + - apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: mssql-app-route + namespace: mssql-persistent + spec: + path: "/" + to: + kind: Service + name: mssql-app-service \ No newline at end of file diff --git a/tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml b/tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml new file mode 100644 index 0000000000..09064697b3 --- /dev/null +++ b/tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: List +items: + - apiVersion: snapshot.storage.k8s.io/v1 + kind: VolumeSnapshotClass + metadata: + name: example-snapclass + labels: + velero.io/csi-volumesnapshot-class: 'true' + annotations: + snapshot.storage.kubernetes.io/is-default-class: 'true' + driver: ebs.csi.aws.com + deletionPolicy: Retain + \ No newline at end of file diff --git a/tests/e2e/velero_helpers.go b/tests/e2e/velero_helpers.go index 42823c5dda..58a2421f03 100755 --- a/tests/e2e/velero_helpers.go +++ b/tests/e2e/velero_helpers.go @@ -24,19 +24,20 @@ import ( ) type veleroCustomResource struct { - Name string - Namespace string - SecretName string - Bucket string - Region string - Provider string - CustomResource *oadpv1alpha1.Velero - Client client.Client + Name string + Namespace string + SecretName string + Bucket string + Region string + Provider string + backupRestoreType string + CustomResource *oadpv1alpha1.Velero + Client client.Client } var veleroPrefix = "velero-e2e-" + string(uuid.NewUUID()) -func (v *veleroCustomResource) Build() error { +func (v *veleroCustomResource) Build(backupRestoreType string) error { // Velero Instance creation spec with backupstorage location default to AWS. Would need to parameterize this later on to support multiple plugins. veleroSpec := oadpv1alpha1.Velero{ ObjectMeta: metav1.ObjectMeta{ @@ -44,7 +45,6 @@ func (v *veleroCustomResource) Build() error { Namespace: v.Namespace, }, Spec: oadpv1alpha1.VeleroSpec{ - EnableRestic: pointer.Bool(true), BackupStorageLocations: []velero.BackupStorageLocationSpec{ { Provider: v.Provider, @@ -64,8 +64,18 @@ func (v *veleroCustomResource) Build() error { oadpv1alpha1.DefaultPluginOpenShift, oadpv1alpha1.DefaultPluginAWS, }, + VeleroFeatureFlags: []string{}, }, } + v.backupRestoreType = backupRestoreType + switch backupRestoreType { + case "restic": + veleroSpec.Spec.EnableRestic = pointer.Bool(true) + case "csi": + veleroSpec.Spec.EnableRestic = pointer.Bool(false) + veleroSpec.Spec.DefaultVeleroPlugins = append(veleroSpec.Spec.DefaultVeleroPlugins, oadpv1alpha1.DefaultPluginCSI) + veleroSpec.Spec.VeleroFeatureFlags = append(veleroSpec.Spec.VeleroFeatureFlags, "EnableCSI") + } v.CustomResource = &veleroSpec return nil } @@ -103,7 +113,7 @@ func (v *veleroCustomResource) Get() (*oadpv1alpha1.Velero, error) { func (v *veleroCustomResource) CreateOrUpdate(spec *oadpv1alpha1.VeleroSpec) error { cr, err := v.Get() if apierrors.IsNotFound(err) { - v.Build() + v.Build(v.backupRestoreType) v.CustomResource.Spec = *spec return v.Create() } @@ -144,7 +154,7 @@ func (v *veleroCustomResource) SetClient() error { return nil } -func getVeleroPods (namespace string) (*corev1.PodList, error){ +func getVeleroPods(namespace string) (*corev1.PodList, error) { clientset, err := setUpClient() if err != nil { return nil, err @@ -204,7 +214,7 @@ func getVeleroContainerLogs(namespace string) (string, error) { } defer podLogs.Close() buf := new(bytes.Buffer) - _, err = io.Copy(buf,podLogs) + _, err = io.Copy(buf, podLogs) if err != nil { return "", err } @@ -219,11 +229,11 @@ func getVeleroContainerFailureLogs(namespace string) []string { log.Printf("cannot get velero container logs") return nil } - containerLogsArray := strings.Split(containerLogs,"\n") + containerLogsArray := strings.Split(containerLogs, "\n") var failureArr = []string{} for i, line := range containerLogsArray { if strings.Contains(line, "level=error") { - failureArr = append(failureArr, fmt.Sprintf("velero container error line#%d: " + line + "\n", i)) + failureArr = append(failureArr, fmt.Sprintf("velero container error line#%d: "+line+"\n", i)) } } return failureArr From f16bb3865a6ab80364e40cd0b36e7b3aa2b8f456 Mon Sep 17 00:00:00 2001 From: hhpatel14 Date: Tue, 2 Nov 2021 15:20:39 -0400 Subject: [PATCH 2/8] code refactoring and, Postbackup-restore varify for MSSQL --- tests/e2e/apps.go | 30 ++++- tests/e2e/backup_restore_suite_test.go | 113 ++++++++---------- .../volumeSnapshotClass.yaml | 0 tests/e2e/velero_helpers.go | 8 +- 4 files changed, 84 insertions(+), 67 deletions(-) rename tests/e2e/sample-applications/{mssql-persistent => gp2-csi}/volumeSnapshotClass.yaml (100%) diff --git a/tests/e2e/apps.go b/tests/e2e/apps.go index c139777098..42ae9baf13 100755 --- a/tests/e2e/apps.go +++ b/tests/e2e/apps.go @@ -7,7 +7,9 @@ import ( "os" "github.com/onsi/ginkgo" - appsv1 "github.com/openshift/api/apps/v1" + ocpappsv1 "github.com/openshift/api/apps/v1" + appsv1 "k8s.io/api/apps/v1" + security "github.com/openshift/api/security/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -102,7 +104,7 @@ func areApplicationPodsRunning(namespace string) wait.ConditionFunc { if len(condition.Message) > 0 { ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Pod not running with condition: %s\n", condition.Message))) } - } + } return false, nil } } @@ -112,7 +114,7 @@ func areApplicationPodsRunning(namespace string) wait.ConditionFunc { func isDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionFunc { return func() (bool, error) { - dc := appsv1.DeploymentConfig{} + dc := ocpappsv1.DeploymentConfig{} err := ocClient.Get(context.Background(), client.ObjectKey{ Namespace: namespace, Name: dcName, @@ -131,3 +133,25 @@ func isDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionF return true, nil } } + +func isDeploymentReady(ocClient client.Client, namespace, dName string) wait.ConditionFunc { + return func() (bool, error) { + deployment := appsv1.Deployment{} + err := ocClient.Get(context.Background(), client.ObjectKey{ + Namespace: namespace, + Name: dName, + }, &deployment) + if err != nil { + return false, err + } + if deployment.Status.AvailableReplicas != deployment.Status.Replicas || deployment.Status.Replicas == 0 { + for _, condition := range deployment.Status.Conditions { + if len(condition.Message) > 0 { + ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("deployment not available with condition: %s\n", condition.Message))) + } + } + return false, errors.New("deployment is not in a ready state") + } + return true, nil + } +} diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 1b2c3a8ede..1d92504f10 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -13,6 +13,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +type BackupRestoreType string + +const ( + csi BackupRestoreType = "csi" + restic BackupRestoreType = "restic" +) + +type VerificationFunction func(client.Client, string) error + var _ = Describe("AWS backup restore tests", func() { var _ = BeforeEach(func() { testSuiteInstanceName := "ts-" + instanceName @@ -31,19 +40,35 @@ var _ = Describe("AWS backup restore tests", func() { }) - type VerificationFunction func(client.Client, string) error - type BackupRestoreCase struct { ApplicationTemplate string ApplicationNamespace string Name string - BackupRestoreType string + BackupRestoreType BackupRestoreType PreBackupVerify VerificationFunction PostRestoreVerify VerificationFunction MaxK8SVersion *k8sVersion MinK8SVersion *k8sVersion } + parksAppReady := VerificationFunction(func(ocClient client.Client, namespace string) error { + Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + return nil + }) + mssqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error { + // This test confirms that SCC restore logic in our plugin is working + Eventually(isDCReady(ocClient, "mssql-persistent", "mssql-deployment"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + Eventually(isDeploymentReady(ocClient, "mssql-persistent", "mssql-app-deployment"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + exists, err := doesSCCExist(ocClient, "mssql-persistent-scc") + if err != nil { + return err + } + if !exists { + return errors.New("did not find MSSQL scc") + } + return nil + }) + DescribeTable("backup and restore applications", func(brCase BackupRestoreCase, expectedErr error) { @@ -56,15 +81,13 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Waiting for velero pod to be running") Eventually(isVeleroPodRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - if brCase.BackupRestoreType == "restic" { - if vel.CustomResource.Spec.EnableRestic == nil || *vel.CustomResource.Spec.EnableRestic { - log.Printf("Waiting for restic pods to be running") - Eventually(areResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - } + if brCase.BackupRestoreType == restic { + log.Printf("Waiting for restic pods to be running") + Eventually(areResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) } - if brCase.BackupRestoreType == "csi" { + if brCase.BackupRestoreType == csi { log.Printf("Creating VolumeSnapshot for CSI backuprestore of %s", brCase.Name) - err = installApplication(vel.Client, "./sample-applications/mssql-persistent/volumeSnapshotClass.yaml") + err = installApplication(vel.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml") Expect(err).ToNot(HaveOccurred()) } @@ -140,79 +163,49 @@ var _ = Describe("AWS backup restore tests", func() { err = uninstallApplication(vel.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) - if brCase.BackupRestoreType == "csi" { + // Wait for namespace to be deleted + Eventually(isNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) + + if brCase.BackupRestoreType == csi { log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name) - err = uninstallApplication(vel.Client, "./sample-applications/mssql-persistent/volumeSnapshotClass.yaml") + err = uninstallApplication(vel.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml") Expect(err).ToNot(HaveOccurred()) } + }, Entry("MSSQL application CSI", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml", ApplicationNamespace: "mssql-persistent", Name: "mssql-e2e", - BackupRestoreType: "csi", - PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - return nil - }), - PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - // This test confirms that SCC restore logic in our plugin is working - exists, err := doesSCCExist(ocClient, "mssql-persistent-scc") - if err != nil { - return err - } - if !exists { - return errors.New("did not find MSSQL scc after restore") - } - return nil - }), + BackupRestoreType: csi, + PreBackupVerify: mssqlReady, + PostRestoreVerify: mssqlReady, }, nil), Entry("Parks application <4.8.0", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/parks-app/manifest.yaml", ApplicationNamespace: "parks-app", Name: "parks-e2e", - BackupRestoreType: "restic", - PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - return nil - }), - PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - return nil - }), - MaxK8SVersion: &k8sVersionOcp47, + BackupRestoreType: restic, + PreBackupVerify: parksAppReady, + PostRestoreVerify: parksAppReady, + MaxK8SVersion: &k8sVersionOcp47, }, nil), Entry("MSSQL application", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-template.yaml", ApplicationNamespace: "mssql-persistent", Name: "mssql-e2e", - BackupRestoreType: "restic", - PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - return nil - }), - PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - // This test confirms that SCC restore logic in our plugin is working - exists, err := doesSCCExist(ocClient, "mssql-persistent-scc") - if err != nil { - return err - } - if !exists { - return errors.New("did not find MSSQL scc after restore") - } - return nil - }), + BackupRestoreType: restic, + PreBackupVerify: mssqlReady, + PostRestoreVerify: mssqlReady, }, nil), Entry("Parks application >=4.8.0", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/parks-app/manifest4.8.yaml", ApplicationNamespace: "parks-app", Name: "parks-e2e", - BackupRestoreType: "restic", - PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - return nil - }), - PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - return nil - }), - MinK8SVersion: &k8sVersionOcp48, + BackupRestoreType: restic, + PreBackupVerify: parksAppReady, + PostRestoreVerify: parksAppReady, + MinK8SVersion: &k8sVersionOcp48, }, nil), ) }) diff --git a/tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml b/tests/e2e/sample-applications/gp2-csi/volumeSnapshotClass.yaml similarity index 100% rename from tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml rename to tests/e2e/sample-applications/gp2-csi/volumeSnapshotClass.yaml diff --git a/tests/e2e/velero_helpers.go b/tests/e2e/velero_helpers.go index 58a2421f03..0ea1eda0dd 100755 --- a/tests/e2e/velero_helpers.go +++ b/tests/e2e/velero_helpers.go @@ -30,14 +30,14 @@ type veleroCustomResource struct { Bucket string Region string Provider string - backupRestoreType string + backupRestoreType BackupRestoreType CustomResource *oadpv1alpha1.Velero Client client.Client } var veleroPrefix = "velero-e2e-" + string(uuid.NewUUID()) -func (v *veleroCustomResource) Build(backupRestoreType string) error { +func (v *veleroCustomResource) Build(backupRestoreType BackupRestoreType) error { // Velero Instance creation spec with backupstorage location default to AWS. Would need to parameterize this later on to support multiple plugins. veleroSpec := oadpv1alpha1.Velero{ ObjectMeta: metav1.ObjectMeta{ @@ -69,9 +69,9 @@ func (v *veleroCustomResource) Build(backupRestoreType string) error { } v.backupRestoreType = backupRestoreType switch backupRestoreType { - case "restic": + case restic: veleroSpec.Spec.EnableRestic = pointer.Bool(true) - case "csi": + case csi: veleroSpec.Spec.EnableRestic = pointer.Bool(false) veleroSpec.Spec.DefaultVeleroPlugins = append(veleroSpec.Spec.DefaultVeleroPlugins, oadpv1alpha1.DefaultPluginCSI) veleroSpec.Spec.VeleroFeatureFlags = append(veleroSpec.Spec.VeleroFeatureFlags, "EnableCSI") From a08b27b7c02d7dd5bb81f9b66cbe2636a41f4855 Mon Sep 17 00:00:00 2001 From: hhpatel14 Date: Thu, 4 Nov 2021 17:02:36 -0400 Subject: [PATCH 3/8] increase waiting time after restore --- tests/e2e/backup_restore_suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 1d92504f10..3726439fa4 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -151,7 +151,7 @@ var _ = Describe("AWS backup restore tests", func() { Expect(succeeded).To(Equal(true)) // verify app is running - Eventually(areApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) + Eventually(areApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) // Run optional custom verification log.Printf("Running post-restore function for case %s", brCase.Name) From 54725eb8b8776fd0fbbe8a4eedcad7585493d84c Mon Sep 17 00:00:00 2001 From: hhpatel14 Date: Mon, 1 Nov 2021 23:35:05 -0400 Subject: [PATCH 4/8] CSI Backrestore E2E test --- tests/e2e/backup_restore_suite_test.go | 81 +++++--- .../mssql-persistent-csi-template.yaml | 177 ++++++++++++++++++ .../mssql-persistent/volumeSnapshotClass.yaml | 14 ++ tests/e2e/velero_helpers.go | 32 ++-- 4 files changed, 271 insertions(+), 33 deletions(-) create mode 100644 tests/e2e/sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml create mode 100644 tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index fe6ace1e67..1b2c3a8ede 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -23,25 +23,6 @@ var _ = Describe("AWS backup restore tests", func() { err = createCredentialsSecret(credData, namespace, credSecretRef) Expect(err).NotTo(HaveOccurred()) - - err = vel.Build() - Expect(err).NotTo(HaveOccurred()) - - err = vel.CreateOrUpdate(&vel.CustomResource.Spec) - Expect(err).NotTo(HaveOccurred()) - - log.Printf("Waiting for velero pod to be running") - Eventually(isVeleroPodRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - - if vel.CustomResource.Spec.EnableRestic == nil || *vel.CustomResource.Spec.EnableRestic { - log.Printf("Waiting for restic pods to be running") - Eventually(areResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - } - - if vel.CustomResource.Spec.BackupImages == nil || *vel.CustomResource.Spec.BackupImages { - log.Printf("Waiting for registry pods to be running") - Eventually(areRegistryDeploymentsAvailable(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - } }) var _ = AfterEach(func() { @@ -56,6 +37,7 @@ var _ = Describe("AWS backup restore tests", func() { ApplicationTemplate string ApplicationNamespace string Name string + BackupRestoreType string PreBackupVerify VerificationFunction PostRestoreVerify VerificationFunction MaxK8SVersion *k8sVersion @@ -64,6 +46,32 @@ var _ = Describe("AWS backup restore tests", func() { DescribeTable("backup and restore applications", func(brCase BackupRestoreCase, expectedErr error) { + + err := vel.Build(brCase.BackupRestoreType) + Expect(err).NotTo(HaveOccurred()) + + err = vel.CreateOrUpdate(&vel.CustomResource.Spec) + Expect(err).NotTo(HaveOccurred()) + + log.Printf("Waiting for velero pod to be running") + Eventually(isVeleroPodRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) + + if brCase.BackupRestoreType == "restic" { + if vel.CustomResource.Spec.EnableRestic == nil || *vel.CustomResource.Spec.EnableRestic { + log.Printf("Waiting for restic pods to be running") + Eventually(areResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) + } + } + if brCase.BackupRestoreType == "csi" { + log.Printf("Creating VolumeSnapshot for CSI backuprestore of %s", brCase.Name) + err = installApplication(vel.Client, "./sample-applications/mssql-persistent/volumeSnapshotClass.yaml") + Expect(err).ToNot(HaveOccurred()) + } + + if vel.CustomResource.Spec.BackupImages == nil || *vel.CustomResource.Spec.BackupImages { + log.Printf("Waiting for registry pods to be running") + Eventually(areRegistryDeploymentsAvailable(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) + } if notVersionTarget, reason := NotServerVersionTarget(brCase.MinK8SVersion, brCase.MaxK8SVersion); notVersionTarget { Skip(reason) } @@ -74,7 +82,7 @@ var _ = Describe("AWS backup restore tests", func() { // install app log.Printf("Installing application for case %s", brCase.Name) - err := installApplication(vel.Client, brCase.ApplicationTemplate) + err = installApplication(vel.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) // wait for pods to be running Eventually(areApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) @@ -131,11 +139,18 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Uninstalling application for case %s", brCase.Name) err = uninstallApplication(vel.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) + + if brCase.BackupRestoreType == "csi" { + log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name) + err = uninstallApplication(vel.Client, "./sample-applications/mssql-persistent/volumeSnapshotClass.yaml") + Expect(err).ToNot(HaveOccurred()) + } }, - Entry("MSSQL application", BackupRestoreCase{ - ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-template.yaml", + Entry("MSSQL application CSI", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml", ApplicationNamespace: "mssql-persistent", Name: "mssql-e2e", + BackupRestoreType: "csi", PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { return nil }), @@ -155,6 +170,7 @@ var _ = Describe("AWS backup restore tests", func() { ApplicationTemplate: "./sample-applications/parks-app/manifest.yaml", ApplicationNamespace: "parks-app", Name: "parks-e2e", + BackupRestoreType: "restic", PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) return nil @@ -164,10 +180,31 @@ var _ = Describe("AWS backup restore tests", func() { }), MaxK8SVersion: &k8sVersionOcp47, }, nil), + Entry("MSSQL application", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-template.yaml", + ApplicationNamespace: "mssql-persistent", + Name: "mssql-e2e", + BackupRestoreType: "restic", + PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { + return nil + }), + PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { + // This test confirms that SCC restore logic in our plugin is working + exists, err := doesSCCExist(ocClient, "mssql-persistent-scc") + if err != nil { + return err + } + if !exists { + return errors.New("did not find MSSQL scc after restore") + } + return nil + }), + }, nil), Entry("Parks application >=4.8.0", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/parks-app/manifest4.8.yaml", ApplicationNamespace: "parks-app", Name: "parks-e2e", + BackupRestoreType: "restic", PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) return nil diff --git a/tests/e2e/sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml b/tests/e2e/sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml new file mode 100644 index 0000000000..0dcd7f394f --- /dev/null +++ b/tests/e2e/sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml @@ -0,0 +1,177 @@ +apiVersion: v1 +kind: List +items: + - kind: Namespace + apiVersion: v1 + metadata: + name: mssql-persistent + labels: + app: mssql + - apiVersion: v1 + kind: Secret + metadata: + name: mssql-secret + namespace: mssql-persistent + labels: + app: mssql + stringData: + mssql-password: P@ssw0rd1! + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: mssql-persistent-sa + namespace: mssql-persistent + labels: + component: mssql-persistent + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: mssql-pvc + namespace: mssql-persistent + labels: + app: mssql + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: gp2-csi + - kind: SecurityContextConstraints + apiVersion: security.openshift.io/v1 + metadata: + name: mssql-persistent-scc + allowPrivilegeEscalation: true + allowPrivilegedContainer: true + runAsUser: + type: RunAsAny + seLinuxContext: + type: RunAsAny + fsGroup: + type: RunAsAny + supplementalGroups: + type: RunAsAny + volumes: + - '*' + users: + - system:admin + - system:serviceaccount:mssql-persistent:mssql-persistent-sa + - apiVersion: apps.openshift.io/v1 + kind: DeploymentConfig + metadata: + name: mssql-deployment + namespace: mssql-persistent + labels: + e2e-app: "true" + app: mssql + spec: + replicas: 1 + selector: + name: mssql + strategy: + type: Recreate + template: + metadata: + labels: + e2e-app: "true" + name: mssql + app: mssql + spec: + serviceAccountName: mssql-persistent-sa + containers: + - env: + - name: ACCEPT_EULA + value: "Y" + - name: SA_PASSWORD + valueFrom: + secretKeyRef: + key: mssql-password + name: mssql-secret + image: quay.io/ocpmigrate/mssql-server:latest + imagePullPolicy: Always + name: mssql + securityContext: + privileged: true + ports: + - containerPort: 1433 + resources: + limits: + memory: "3Gi" + cpu: "0.5" + requests: + memory: "3Gi" + cpu: "0.5" + volumeMounts: + - mountPath: "/var/opt/mssql/data" + name: mssql-vol + volumes: + - name: mssql-vol + persistentVolumeClaim: + claimName: mssql-pvc + - apiVersion: v1 + kind: Service + metadata: + name: mssql-service + namespace: mssql-persistent + spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 + type: ClusterIP + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: mssql-app-deployment + namespace: mssql-persistent + labels: + e2e-app: "true" + spec: + replicas: 1 + selector: + matchLabels: + app: mssql-app + template: + metadata: + labels: + e2e-app: "true" + app: mssql-app + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: mssql-persistent-sa + containers: + - name: mssql-app + image: quay.io/ocpmigrate/mssql-sample-app:microsoft + imagePullPolicy: Always + ports: + - containerPort: 5000 + securityContext: + privileged: true + env: + - name: ConnString + value: "Server=mssql-service.mssql-persistent.svc.cluster.local;Database=ProductCatalog;User ID=WebLogin; password=SQLPass1234!" + - apiVersion: v1 + kind: Service + metadata: + name: mssql-app-service + namespace: mssql-persistent + spec: + selector: + app: mssql-app + ports: + - name: "tcp" + protocol: TCP + port: 5000 + targetPort: 5000 + - apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: mssql-app-route + namespace: mssql-persistent + spec: + path: "/" + to: + kind: Service + name: mssql-app-service \ No newline at end of file diff --git a/tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml b/tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml new file mode 100644 index 0000000000..09064697b3 --- /dev/null +++ b/tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: List +items: + - apiVersion: snapshot.storage.k8s.io/v1 + kind: VolumeSnapshotClass + metadata: + name: example-snapclass + labels: + velero.io/csi-volumesnapshot-class: 'true' + annotations: + snapshot.storage.kubernetes.io/is-default-class: 'true' + driver: ebs.csi.aws.com + deletionPolicy: Retain + \ No newline at end of file diff --git a/tests/e2e/velero_helpers.go b/tests/e2e/velero_helpers.go index ec44035a94..598e646a5a 100755 --- a/tests/e2e/velero_helpers.go +++ b/tests/e2e/velero_helpers.go @@ -25,19 +25,20 @@ import ( ) type veleroCustomResource struct { - Name string - Namespace string - SecretName string - Bucket string - Region string - Provider string - CustomResource *oadpv1alpha1.Velero - Client client.Client + Name string + Namespace string + SecretName string + Bucket string + Region string + Provider string + backupRestoreType string + CustomResource *oadpv1alpha1.Velero + Client client.Client } var veleroPrefix = "velero-e2e-" + string(uuid.NewUUID()) -func (v *veleroCustomResource) Build() error { +func (v *veleroCustomResource) Build(backupRestoreType string) error { // Velero Instance creation spec with backupstorage location default to AWS. Would need to parameterize this later on to support multiple plugins. veleroSpec := oadpv1alpha1.Velero{ ObjectMeta: metav1.ObjectMeta{ @@ -45,7 +46,6 @@ func (v *veleroCustomResource) Build() error { Namespace: v.Namespace, }, Spec: oadpv1alpha1.VeleroSpec{ - EnableRestic: pointer.Bool(true), BackupStorageLocations: []velero.BackupStorageLocationSpec{ { Provider: v.Provider, @@ -65,8 +65,18 @@ func (v *veleroCustomResource) Build() error { oadpv1alpha1.DefaultPluginOpenShift, oadpv1alpha1.DefaultPluginAWS, }, + VeleroFeatureFlags: []string{}, }, } + v.backupRestoreType = backupRestoreType + switch backupRestoreType { + case "restic": + veleroSpec.Spec.EnableRestic = pointer.Bool(true) + case "csi": + veleroSpec.Spec.EnableRestic = pointer.Bool(false) + veleroSpec.Spec.DefaultVeleroPlugins = append(veleroSpec.Spec.DefaultVeleroPlugins, oadpv1alpha1.DefaultPluginCSI) + veleroSpec.Spec.VeleroFeatureFlags = append(veleroSpec.Spec.VeleroFeatureFlags, "EnableCSI") + } v.CustomResource = &veleroSpec return nil } @@ -104,7 +114,7 @@ func (v *veleroCustomResource) Get() (*oadpv1alpha1.Velero, error) { func (v *veleroCustomResource) CreateOrUpdate(spec *oadpv1alpha1.VeleroSpec) error { cr, err := v.Get() if apierrors.IsNotFound(err) { - v.Build() + v.Build(v.backupRestoreType) v.CustomResource.Spec = *spec return v.Create() } From 3d222c36aa14a7b87de5dab26a928c9e1818d1f2 Mon Sep 17 00:00:00 2001 From: hhpatel14 Date: Tue, 2 Nov 2021 15:20:39 -0400 Subject: [PATCH 5/8] code refactoring and, Postbackup-restore varify for MSSQL --- tests/e2e/apps.go | 30 ++++- tests/e2e/backup_restore_suite_test.go | 113 ++++++++---------- .../volumeSnapshotClass.yaml | 0 tests/e2e/velero_helpers.go | 8 +- 4 files changed, 84 insertions(+), 67 deletions(-) rename tests/e2e/sample-applications/{mssql-persistent => gp2-csi}/volumeSnapshotClass.yaml (100%) diff --git a/tests/e2e/apps.go b/tests/e2e/apps.go index c139777098..42ae9baf13 100755 --- a/tests/e2e/apps.go +++ b/tests/e2e/apps.go @@ -7,7 +7,9 @@ import ( "os" "github.com/onsi/ginkgo" - appsv1 "github.com/openshift/api/apps/v1" + ocpappsv1 "github.com/openshift/api/apps/v1" + appsv1 "k8s.io/api/apps/v1" + security "github.com/openshift/api/security/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -102,7 +104,7 @@ func areApplicationPodsRunning(namespace string) wait.ConditionFunc { if len(condition.Message) > 0 { ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Pod not running with condition: %s\n", condition.Message))) } - } + } return false, nil } } @@ -112,7 +114,7 @@ func areApplicationPodsRunning(namespace string) wait.ConditionFunc { func isDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionFunc { return func() (bool, error) { - dc := appsv1.DeploymentConfig{} + dc := ocpappsv1.DeploymentConfig{} err := ocClient.Get(context.Background(), client.ObjectKey{ Namespace: namespace, Name: dcName, @@ -131,3 +133,25 @@ func isDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionF return true, nil } } + +func isDeploymentReady(ocClient client.Client, namespace, dName string) wait.ConditionFunc { + return func() (bool, error) { + deployment := appsv1.Deployment{} + err := ocClient.Get(context.Background(), client.ObjectKey{ + Namespace: namespace, + Name: dName, + }, &deployment) + if err != nil { + return false, err + } + if deployment.Status.AvailableReplicas != deployment.Status.Replicas || deployment.Status.Replicas == 0 { + for _, condition := range deployment.Status.Conditions { + if len(condition.Message) > 0 { + ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("deployment not available with condition: %s\n", condition.Message))) + } + } + return false, errors.New("deployment is not in a ready state") + } + return true, nil + } +} diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 1b2c3a8ede..1d92504f10 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -13,6 +13,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +type BackupRestoreType string + +const ( + csi BackupRestoreType = "csi" + restic BackupRestoreType = "restic" +) + +type VerificationFunction func(client.Client, string) error + var _ = Describe("AWS backup restore tests", func() { var _ = BeforeEach(func() { testSuiteInstanceName := "ts-" + instanceName @@ -31,19 +40,35 @@ var _ = Describe("AWS backup restore tests", func() { }) - type VerificationFunction func(client.Client, string) error - type BackupRestoreCase struct { ApplicationTemplate string ApplicationNamespace string Name string - BackupRestoreType string + BackupRestoreType BackupRestoreType PreBackupVerify VerificationFunction PostRestoreVerify VerificationFunction MaxK8SVersion *k8sVersion MinK8SVersion *k8sVersion } + parksAppReady := VerificationFunction(func(ocClient client.Client, namespace string) error { + Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + return nil + }) + mssqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error { + // This test confirms that SCC restore logic in our plugin is working + Eventually(isDCReady(ocClient, "mssql-persistent", "mssql-deployment"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + Eventually(isDeploymentReady(ocClient, "mssql-persistent", "mssql-app-deployment"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + exists, err := doesSCCExist(ocClient, "mssql-persistent-scc") + if err != nil { + return err + } + if !exists { + return errors.New("did not find MSSQL scc") + } + return nil + }) + DescribeTable("backup and restore applications", func(brCase BackupRestoreCase, expectedErr error) { @@ -56,15 +81,13 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Waiting for velero pod to be running") Eventually(isVeleroPodRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - if brCase.BackupRestoreType == "restic" { - if vel.CustomResource.Spec.EnableRestic == nil || *vel.CustomResource.Spec.EnableRestic { - log.Printf("Waiting for restic pods to be running") - Eventually(areResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - } + if brCase.BackupRestoreType == restic { + log.Printf("Waiting for restic pods to be running") + Eventually(areResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) } - if brCase.BackupRestoreType == "csi" { + if brCase.BackupRestoreType == csi { log.Printf("Creating VolumeSnapshot for CSI backuprestore of %s", brCase.Name) - err = installApplication(vel.Client, "./sample-applications/mssql-persistent/volumeSnapshotClass.yaml") + err = installApplication(vel.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml") Expect(err).ToNot(HaveOccurred()) } @@ -140,79 +163,49 @@ var _ = Describe("AWS backup restore tests", func() { err = uninstallApplication(vel.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) - if brCase.BackupRestoreType == "csi" { + // Wait for namespace to be deleted + Eventually(isNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) + + if brCase.BackupRestoreType == csi { log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name) - err = uninstallApplication(vel.Client, "./sample-applications/mssql-persistent/volumeSnapshotClass.yaml") + err = uninstallApplication(vel.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml") Expect(err).ToNot(HaveOccurred()) } + }, Entry("MSSQL application CSI", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml", ApplicationNamespace: "mssql-persistent", Name: "mssql-e2e", - BackupRestoreType: "csi", - PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - return nil - }), - PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - // This test confirms that SCC restore logic in our plugin is working - exists, err := doesSCCExist(ocClient, "mssql-persistent-scc") - if err != nil { - return err - } - if !exists { - return errors.New("did not find MSSQL scc after restore") - } - return nil - }), + BackupRestoreType: csi, + PreBackupVerify: mssqlReady, + PostRestoreVerify: mssqlReady, }, nil), Entry("Parks application <4.8.0", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/parks-app/manifest.yaml", ApplicationNamespace: "parks-app", Name: "parks-e2e", - BackupRestoreType: "restic", - PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - return nil - }), - PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - return nil - }), - MaxK8SVersion: &k8sVersionOcp47, + BackupRestoreType: restic, + PreBackupVerify: parksAppReady, + PostRestoreVerify: parksAppReady, + MaxK8SVersion: &k8sVersionOcp47, }, nil), Entry("MSSQL application", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-template.yaml", ApplicationNamespace: "mssql-persistent", Name: "mssql-e2e", - BackupRestoreType: "restic", - PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - return nil - }), - PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - // This test confirms that SCC restore logic in our plugin is working - exists, err := doesSCCExist(ocClient, "mssql-persistent-scc") - if err != nil { - return err - } - if !exists { - return errors.New("did not find MSSQL scc after restore") - } - return nil - }), + BackupRestoreType: restic, + PreBackupVerify: mssqlReady, + PostRestoreVerify: mssqlReady, }, nil), Entry("Parks application >=4.8.0", BackupRestoreCase{ ApplicationTemplate: "./sample-applications/parks-app/manifest4.8.yaml", ApplicationNamespace: "parks-app", Name: "parks-e2e", - BackupRestoreType: "restic", - PreBackupVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - Eventually(isDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - return nil - }), - PostRestoreVerify: VerificationFunction(func(ocClient client.Client, namespace string) error { - return nil - }), - MinK8SVersion: &k8sVersionOcp48, + BackupRestoreType: restic, + PreBackupVerify: parksAppReady, + PostRestoreVerify: parksAppReady, + MinK8SVersion: &k8sVersionOcp48, }, nil), ) }) diff --git a/tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml b/tests/e2e/sample-applications/gp2-csi/volumeSnapshotClass.yaml similarity index 100% rename from tests/e2e/sample-applications/mssql-persistent/volumeSnapshotClass.yaml rename to tests/e2e/sample-applications/gp2-csi/volumeSnapshotClass.yaml diff --git a/tests/e2e/velero_helpers.go b/tests/e2e/velero_helpers.go index 598e646a5a..0fa5e96185 100755 --- a/tests/e2e/velero_helpers.go +++ b/tests/e2e/velero_helpers.go @@ -31,14 +31,14 @@ type veleroCustomResource struct { Bucket string Region string Provider string - backupRestoreType string + backupRestoreType BackupRestoreType CustomResource *oadpv1alpha1.Velero Client client.Client } var veleroPrefix = "velero-e2e-" + string(uuid.NewUUID()) -func (v *veleroCustomResource) Build(backupRestoreType string) error { +func (v *veleroCustomResource) Build(backupRestoreType BackupRestoreType) error { // Velero Instance creation spec with backupstorage location default to AWS. Would need to parameterize this later on to support multiple plugins. veleroSpec := oadpv1alpha1.Velero{ ObjectMeta: metav1.ObjectMeta{ @@ -70,9 +70,9 @@ func (v *veleroCustomResource) Build(backupRestoreType string) error { } v.backupRestoreType = backupRestoreType switch backupRestoreType { - case "restic": + case restic: veleroSpec.Spec.EnableRestic = pointer.Bool(true) - case "csi": + case csi: veleroSpec.Spec.EnableRestic = pointer.Bool(false) veleroSpec.Spec.DefaultVeleroPlugins = append(veleroSpec.Spec.DefaultVeleroPlugins, oadpv1alpha1.DefaultPluginCSI) veleroSpec.Spec.VeleroFeatureFlags = append(veleroSpec.Spec.VeleroFeatureFlags, "EnableCSI") From 05db25cd20730d693f67ec5782ed9af57b2d9f3c Mon Sep 17 00:00:00 2001 From: hhpatel14 Date: Thu, 4 Nov 2021 17:02:36 -0400 Subject: [PATCH 6/8] increase waiting time after restore --- tests/e2e/backup_restore_suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 1d92504f10..3726439fa4 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -151,7 +151,7 @@ var _ = Describe("AWS backup restore tests", func() { Expect(succeeded).To(Equal(true)) // verify app is running - Eventually(areApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) + Eventually(areApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) // Run optional custom verification log.Printf("Running post-restore function for case %s", brCase.Name) From f2b9b334af3724ad185b7bb1d0ece499f47dab37 Mon Sep 17 00:00:00 2001 From: hhpatel14 Date: Mon, 8 Nov 2021 18:25:47 -0500 Subject: [PATCH 7/8] remove vel.build from e2e_suit_test.go --- tests/e2e/e2e_suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index 98cf4206c8..a13a18f461 100755 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -69,7 +69,7 @@ var _ = BeforeSuite(func() { vel.Name = testSuiteInstanceName vel.SetClient() - vel.Build() + //vel.Build() Expect(doesNamespaceExist(namespace)).Should(BeTrue()) }) From 64c4c80bb8e8b5bace3bdba0b34ece33a6cf920c Mon Sep 17 00:00:00 2001 From: Savitha Raghunathan Date: Fri, 12 Nov 2021 11:03:08 -0800 Subject: [PATCH 8/8] Adding changes to velero deployment suite to support BackupRestoreType --- tests/e2e/backup_restore_suite_test.go | 7 ---- tests/e2e/e2e_suite_test.go | 1 - tests/e2e/velero_deployment_suite_test.go | 43 +++++++++++++++-------- tests/e2e/velero_helpers.go | 7 ++++ 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 3726439fa4..0dc2f03f05 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -13,13 +13,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type BackupRestoreType string - -const ( - csi BackupRestoreType = "csi" - restic BackupRestoreType = "restic" -) - type VerificationFunction func(client.Client, string) error var _ = Describe("AWS backup restore tests", func() { diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index a13a18f461..268c214df3 100755 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -69,7 +69,6 @@ var _ = BeforeSuite(func() { vel.Name = testSuiteInstanceName vel.SetClient() - //vel.Build() Expect(doesNamespaceExist(namespace)).Should(BeTrue()) }) diff --git a/tests/e2e/velero_deployment_suite_test.go b/tests/e2e/velero_deployment_suite_test.go index b2c000e7b2..466450ef79 100644 --- a/tests/e2e/velero_deployment_suite_test.go +++ b/tests/e2e/velero_deployment_suite_test.go @@ -16,14 +16,20 @@ import ( var _ = Describe("Configuration testing for Velero Custom Resource", func() { type InstallCase struct { - Name string - VeleroSpec *oadpv1alpha1.VeleroSpec - WantError bool + Name string + BRestoreType BackupRestoreType + VeleroSpec *oadpv1alpha1.VeleroSpec + WantError bool } DescribeTable("Updating custom resource with new configuration", func(installCase InstallCase, expectedErr error) { - err := vel.CreateOrUpdate(installCase.VeleroSpec) + //TODO: Calling vel.build() is the old pattern. + //Change it later to make sure all the spec values are passed for every test case, + // instead of assigning the values in advance to the Velero CR + err := vel.Build(installCase.BRestoreType) + Expect(err).NotTo(HaveOccurred()) + err = vel.CreateOrUpdate(installCase.VeleroSpec) Expect(err).ToNot(HaveOccurred()) log.Printf("Waiting for velero pod to be running") Eventually(isVeleroPodRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) @@ -79,7 +85,8 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { }, Entry("Default velero CR", InstallCase{ - Name: "default-cr", + Name: "default-cr", + BRestoreType: restic, VeleroSpec: &oadpv1alpha1.VeleroSpec{ EnableRestic: pointer.Bool(true), BackupStorageLocations: []velero.BackupStorageLocationSpec{ @@ -105,7 +112,8 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { WantError: false, }, nil), Entry("Velero CR with bsl and vsl", InstallCase{ - Name: "default-cr-bsl-vsl", + Name: "default-cr-bsl-vsl", + BRestoreType: "restic", VeleroSpec: &oadpv1alpha1.VeleroSpec{ EnableRestic: pointer.Bool(true), BackupStorageLocations: []velero.BackupStorageLocationSpec{ @@ -139,7 +147,8 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { WantError: false, }, nil), Entry("Velero CR with bsl and multiple vsl", InstallCase{ - Name: "default-cr-bsl-vsl", + Name: "default-cr-bsl-vsl", + BRestoreType: "restic", VeleroSpec: &oadpv1alpha1.VeleroSpec{ EnableRestic: pointer.Bool(true), BackupStorageLocations: []velero.BackupStorageLocationSpec{ @@ -179,7 +188,8 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { WantError: false, }, nil), Entry("Velero CR with no bsl and multiple vsl", InstallCase{ - Name: "default-cr-multiple-vsl", + Name: "default-cr-multiple-vsl", + BRestoreType: "restic", VeleroSpec: &oadpv1alpha1.VeleroSpec{ EnableRestic: pointer.Bool(true), VolumeSnapshotLocations: []velero.VolumeSnapshotLocationSpec{ @@ -204,9 +214,10 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { WantError: false, }, nil), Entry("Default velero CR with restic disabled", InstallCase{ - Name: "default-cr-no-restic", + Name: "default-cr-no-restic", + BRestoreType: "restic", VeleroSpec: &oadpv1alpha1.VeleroSpec{ - EnableRestic: pointer.Bool(false), + EnableRestic: pointer.Bool(true), BackupStorageLocations: []velero.BackupStorageLocationSpec{ { Provider: provider, @@ -230,7 +241,8 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { WantError: false, }, nil), Entry("Adding CSI plugin", InstallCase{ - Name: "default-cr-csi", + Name: "default-cr-csi", + BRestoreType: "restic", VeleroSpec: &oadpv1alpha1.VeleroSpec{ EnableRestic: pointer.Bool(true), BackupStorageLocations: []velero.BackupStorageLocationSpec{ @@ -257,7 +269,8 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { WantError: false, }, nil), Entry("Set restic node selector", InstallCase{ - Name: "default-cr-node-selector", + Name: "default-cr-node-selector", + BRestoreType: "restic", VeleroSpec: &oadpv1alpha1.VeleroSpec{ EnableRestic: pointer.Bool(true), ResticNodeSelector: map[string]string{ @@ -287,7 +300,8 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { WantError: false, }, nil), Entry("Enable tolerations", InstallCase{ - Name: "default-cr-tolerations", + Name: "default-cr-tolerations", + BRestoreType: "restic", VeleroSpec: &oadpv1alpha1.VeleroSpec{ EnableRestic: pointer.Bool(true), VeleroTolerations: []corev1.Toleration{ @@ -322,7 +336,8 @@ var _ = Describe("Configuration testing for Velero Custom Resource", func() { WantError: false, }, nil), Entry("NoDefaultBackupLocation", InstallCase{ - Name: "default-cr-node-selector", + Name: "default-cr-node-selector", + BRestoreType: "restic", VeleroSpec: &oadpv1alpha1.VeleroSpec{ EnableRestic: pointer.Bool(true), BackupStorageLocations: []velero.BackupStorageLocationSpec{}, diff --git a/tests/e2e/velero_helpers.go b/tests/e2e/velero_helpers.go index 0fa5e96185..1d131af6c7 100755 --- a/tests/e2e/velero_helpers.go +++ b/tests/e2e/velero_helpers.go @@ -24,6 +24,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" ) +type BackupRestoreType string + +const ( + csi BackupRestoreType = "csi" + restic BackupRestoreType = "restic" +) + type veleroCustomResource struct { Name string Namespace string