From d68aa1e3aef225c6e14de963eb9725846d27321b Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Tue, 10 Mar 2026 10:14:10 +0100 Subject: [PATCH 01/12] Extract cloud storage --- .../c3/appframework_aws_test.go | 426 +------------ .../c3/appframework_azure_test.go | 423 +------------ .../c3/appframework_gcs_test.go | 569 +----------------- test/testenv/cloud_storage_backend.go | 105 ++++ 4 files changed, 124 insertions(+), 1399 deletions(-) create mode 100644 test/testenv/cloud_storage_backend.go diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework_aws/c3/appframework_aws_test.go index 2d150f5ac..5b29a7eb9 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework_aws/c3/appframework_aws_test.go @@ -49,6 +49,7 @@ var _ = Describe("c3appfw test", func() { var appSourceNameShc string var uploadedApps []string var filePresentOnOperator bool + var backend testenv.CloudStorageBackend ctx := context.TODO() @@ -59,6 +60,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) Expect(err).To(Succeed(), "Unable to create testcaseenv") testenv.SpecifiedTestTimeout = 4000 + backend = testenv.NewCloudStorageBackend(testS3Bucket, testDataS3Bucket) deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") }) @@ -76,9 +78,9 @@ var _ = Describe("c3appfw test", func() { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) } - // Delete files uploaded to S3 + // Delete files uploaded to cloud storage if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) } if filePresentOnOperator { @@ -91,429 +93,13 @@ var _ = Describe("c3appfw test", func() { Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { It("smoke, c3, masterappframeworkc3, appframework: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to S3 for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to S3 for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on S3 - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to S3 for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to S3 for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on S3 - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to S3 for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to S3 for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - + testenv.C3AppFrameworkUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { It("smoke, c3, masterappframeworkc3, appframework: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to S3 for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to S3 for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Master and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on S3 - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to S3 for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to S3 for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on S3 - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to S3 for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to S3 for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + testenv.C3AppFrameworkDowngradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go index c7fea6ff3..f6fc5bf5b 100644 --- a/test/appframework_az/c3/appframework_azure_test.go +++ b/test/appframework_az/c3/appframework_azure_test.go @@ -45,6 +45,7 @@ var _ = Describe("c3appfw test", func() { var appSourceNameShc string var uploadedApps []string var filePresentOnOperator bool + var backend testenv.CloudStorageBackend ctx := context.TODO() @@ -53,6 +54,7 @@ var _ = Describe("c3appfw test", func() { name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) Expect(err).To(Succeed(), "Unable to create testcaseenv") + backend = testenv.NewCloudStorageBackend("", "") deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") }) @@ -70,10 +72,9 @@ var _ = Describe("c3appfw test", func() { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) } - // Delete files uploaded to Azure + // Delete files uploaded to cloud storage if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) } if filePresentOnOperator { @@ -86,425 +87,13 @@ var _ = Describe("c3appfw test", func() { Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - //uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - + testenv.C3AppFrameworkUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + testenv.C3AppFrameworkDowngradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go index 6a39fbcff..503b54b10 100644 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_test.go @@ -15,24 +15,16 @@ package c3gcpappfw import ( "context" - //"encoding/json" "fmt" "path/filepath" - //"strings" - //"time" - - //enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" - //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) var _ = Describe("c3appfw test", func() { @@ -50,6 +42,7 @@ var _ = Describe("c3appfw test", func() { var appSourceNameShc string var uploadedApps []string var filePresentOnOperator bool + var backend testenv.CloudStorageBackend ctx := context.TODO() @@ -60,6 +53,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) Expect(err).To(Succeed(), "Unable to create testcaseenv") testenv.SpecifiedTestTimeout = 5000 + backend = testenv.NewCloudStorageBackend(testGcsBucket, testDataGcsBucket) deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") }) @@ -77,9 +71,9 @@ var _ = Describe("c3appfw test", func() { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) } - // Delete files uploaded to GCS + // Delete files uploaded to cloud storage if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) } if filePresentOnOperator { @@ -92,568 +86,19 @@ var _ = Describe("c3appfw test", func() { Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCS - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Bucket for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to GCS for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - + testenv.C3AppFrameworkUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Master and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on GCS - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to GCS for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to GCS for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + testenv.C3AppFrameworkDowngradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS - * Create app source with local scope for C3 SVA (Cluster Master and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCS for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCS - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + testenv.C3AppFrameworkLocalScopeUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) diff --git a/test/testenv/cloud_storage_backend.go b/test/testenv/cloud_storage_backend.go new file mode 100644 index 000000000..b25587b67 --- /dev/null +++ b/test/testenv/cloud_storage_backend.go @@ -0,0 +1,105 @@ +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" +) + +// CloudStorageBackend abstracts cloud-specific upload/delete/download +// operations so that app framework integration tests can be written +// once and parameterized by provider. +type CloudStorageBackend interface { + UploadFiles(ctx context.Context, testDir string, appFileList []string, localDir string) ([]string, error) + DeleteFiles(ctx context.Context, uploadedFiles []string) error + DownloadFiles(ctx context.Context, remoteDir string, localDir string, fileList []string) error + GetCloudProvider() string +} + +// NewCloudStorageBackend returns the correct CloudStorageBackend implementation +// based on the current ClusterProvider. The bucket and dataBucket parameters +// are used by S3 and GCP; Azure ignores them. +func NewCloudStorageBackend(bucket, dataBucket string) CloudStorageBackend { + switch ClusterProvider { + case "eks": + return NewS3Backend(bucket, dataBucket) + case "azure": + return NewAzureBackend() + case "gcp": + return NewGCPBackend(bucket, dataBucket) + default: + Fail(fmt.Sprintf("unsupported cluster provider: %s", ClusterProvider)) + return nil + } +} + +// S3Backend implements CloudStorageBackend for AWS S3. +type S3Backend struct { + Bucket string + DataBucket string +} + +func NewS3Backend(bucket, dataBucket string) *S3Backend { + return &S3Backend{Bucket: bucket, DataBucket: dataBucket} +} + +func (b *S3Backend) UploadFiles(_ context.Context, testDir string, appFileList []string, localDir string) ([]string, error) { + return UploadFilesToS3(b.Bucket, testDir, appFileList, localDir) +} + +func (b *S3Backend) DeleteFiles(_ context.Context, uploadedFiles []string) error { + return DeleteFilesOnS3(b.Bucket, uploadedFiles) +} + +func (b *S3Backend) DownloadFiles(_ context.Context, remoteDir string, localDir string, fileList []string) error { + return DownloadFilesFromS3(b.DataBucket, remoteDir, localDir, fileList) +} + +func (b *S3Backend) GetCloudProvider() string { return "eks" } + +// AzureBackend implements CloudStorageBackend for Azure Blob Storage. +type AzureBackend struct{} + +func NewAzureBackend() *AzureBackend { + return &AzureBackend{} +} + +func (b *AzureBackend) UploadFiles(ctx context.Context, testDir string, appFileList []string, localDir string) ([]string, error) { + return UploadFilesToAzure(ctx, StorageAccount, StorageAccountKey, localDir, testDir, appFileList) +} + +func (b *AzureBackend) DeleteFiles(ctx context.Context, uploadedFiles []string) error { + client := &AzureBlobClient{} + return client.DeleteFilesOnAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, uploadedFiles) +} + +func (b *AzureBackend) DownloadFiles(ctx context.Context, remoteDir string, localDir string, fileList []string) error { + return DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, localDir, remoteDir, fileList) +} + +func (b *AzureBackend) GetCloudProvider() string { return "azure" } + +// GCPBackend implements CloudStorageBackend for Google Cloud Storage. +type GCPBackend struct { + Bucket string + DataBucket string +} + +func NewGCPBackend(bucket, dataBucket string) *GCPBackend { + return &GCPBackend{Bucket: bucket, DataBucket: dataBucket} +} + +func (b *GCPBackend) UploadFiles(_ context.Context, testDir string, appFileList []string, localDir string) ([]string, error) { + return UploadFilesToGCP(b.Bucket, testDir, appFileList, localDir) +} + +func (b *GCPBackend) DeleteFiles(_ context.Context, uploadedFiles []string) error { + return DeleteFilesOnGCP(b.Bucket, uploadedFiles) +} + +func (b *GCPBackend) DownloadFiles(_ context.Context, remoteDir string, localDir string, fileList []string) error { + return DownloadFilesFromGCP(b.DataBucket, remoteDir, localDir, fileList) +} + +func (b *GCPBackend) GetCloudProvider() string { return "gcp" } From 8e958cf9fdb88ca2cd64f91ef46a4adda7d9472f Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Tue, 10 Mar 2026 10:24:45 +0100 Subject: [PATCH 02/12] add test flows --- test/testenv/appframework_c3_testflows.go | 412 ++++++++++++++++++++++ 1 file changed, 412 insertions(+) create mode 100644 test/testenv/appframework_c3_testflows.go diff --git a/test/testenv/appframework_c3_testflows.go b/test/testenv/appframework_c3_testflows.go new file mode 100644 index 000000000..f573c7463 --- /dev/null +++ b/test/testenv/appframework_c3_testflows.go @@ -0,0 +1,412 @@ +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/splunk/enterprise" + corev1 "k8s.io/api/core/v1" +) + +// C3AppFrameworkUpgradeTest deploys a C3 SVA with App Framework enabled, +// installs V1 apps (with MC), then upgrades them to V2 and re-verifies. +// This is the shared flow behind the "upgrade" smoke test in every cloud provider. +func C3AppFrameworkUpgradeTest( + ctx context.Context, + backend CloudStorageBackend, + testcaseEnvInst *TestCaseEnv, + deployment *Deployment, + appListV1, appListV2 []string, + downloadDirV1, downloadDirV2 string, + uploadedApps *[]string, +) { + appVersion := "V1" + appFileList := GetAppFileList(appListV1) + + // Upload V1 apps for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) + testDirMC := "c3appfw-mc-" + RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + RandomDNSName(3) + appSourceVolumeNameMC := "appframework-test-volume-mc-" + RandomDNSName(3) + appFrameworkSpecMC := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) + + mcSpec := enterpriseApi.MonitoringConsoleSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + AppFrameworkConfig: appFrameworkSpecMC, + } + + testcaseEnvInst.Log.Info("Deploy Monitoring Console") + mcName := deployment.GetName() + mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + // Upload V1 apps for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) + testDirIdxc := "c3appfw-idxc-" + RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + // Upload V1 apps for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + testDirShc := "c3appfw-shc-" + RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + RandomDNSName(3) + appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + RandomDNSName(3) + appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + RandomDNSName(3) + appSourceVolumeNameShc := "appframework-test-volume-shc-" + RandomDNSName(3) + appFrameworkSpecIdxc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) + + resourceVersion := GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) + + testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") + indexerReplicas := 3 + shReplicas := 3 + cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") + + ClusterMasterReady(ctx, deployment, testcaseEnvInst) + SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) + VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) + + splunkPodAge := GetPodsStartTime(testcaseEnvInst.GetName()) + + testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") + ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) + _, err = GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) + Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) + scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} + allPods := DumpGetPods(testcaseEnvInst.GetName()) + VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + + // Initial verifications + idxcPodNames := GeneratePodNameSlice(IndexerPod, deployment.GetName(), indexerReplicas, false, 1) + shcPodNames := GeneratePodNameSlice(SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) + cmPod := []string{fmt.Sprintf(ClusterMasterPod, deployment.GetName())} + deployerPod := []string{fmt.Sprintf(DeployerPod, deployment.GetName())} + mcPod := []string{fmt.Sprintf(MonitoringConsolePod, deployment.GetName())} + cmAppSourceInfo := AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} + shcAppSourceInfo := AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} + mcAppSourceInfo := AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} + allAppSourceInfo := []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} + ClusterMasterBundleHash := AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + + // Upgrade apps + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) + backend.DeleteFiles(ctx, *uploadedApps) + *uploadedApps = nil + + resourceVersion = GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) + + appVersion = "V2" + appFileList = GetAppFileList(appListV2) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + + ClusterMasterReady(ctx, deployment, testcaseEnvInst) + SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) + VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + splunkPodAge = GetPodsStartTime(testcaseEnvInst.GetName()) + + // Final verifications + cmAppSourceInfo.CrAppVersion = appVersion + cmAppSourceInfo.CrAppList = appListV2 + cmAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) + shcAppSourceInfo.CrAppVersion = appVersion + shcAppSourceInfo.CrAppList = appListV2 + shcAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) + mcAppSourceInfo.CrAppVersion = appVersion + mcAppSourceInfo.CrAppList = appListV2 + mcAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) + allAppSourceInfo = []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} + AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) + + VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) +} + +// C3AppFrameworkDowngradeTest deploys a C3 SVA with App Framework enabled, +// installs V2 apps (with MC), then downgrades them to V1 and re-verifies. +func C3AppFrameworkDowngradeTest( + ctx context.Context, + backend CloudStorageBackend, + testcaseEnvInst *TestCaseEnv, + deployment *Deployment, + appListV1, appListV2 []string, + downloadDirV1, downloadDirV2 string, + uploadedApps *[]string, +) { + appVersion := "V2" + appFileList := GetAppFileList(appListV2) + + // Upload V2 apps for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) + testDirMC := "c3appfw-mc-" + RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + RandomDNSName(3) + appSourceVolumeNameMC := "appframework-test-volume-mc-" + RandomDNSName(3) + appFrameworkSpecMC := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) + + mcSpec := enterpriseApi.MonitoringConsoleSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + AppFrameworkConfig: appFrameworkSpecMC, + } + + testcaseEnvInst.Log.Info("Deploy Monitoring Console") + mcName := deployment.GetName() + mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + // Upload V2 apps for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) + testDirIdxc := "c3appfw-idxc-" + RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + // Upload V2 apps for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + testDirShc := "c3appfw-shc-" + RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + RandomDNSName(3) + appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + RandomDNSName(3) + appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + RandomDNSName(3) + appSourceVolumeNameShc := "appframework-test-volume-shc-" + RandomDNSName(3) + appFrameworkSpecIdxc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) + + resourceVersion := GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) + + testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") + indexerReplicas := 3 + shReplicas := 3 + cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") + + ClusterMasterReady(ctx, deployment, testcaseEnvInst) + SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) + VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + splunkPodAge := GetPodsStartTime(testcaseEnvInst.GetName()) + + // Initial verifications + idxcPodNames := GeneratePodNameSlice(IndexerPod, deployment.GetName(), indexerReplicas, false, 1) + shcPodNames := GeneratePodNameSlice(SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) + cmPod := []string{fmt.Sprintf(ClusterMasterPod, deployment.GetName())} + deployerPod := []string{fmt.Sprintf(DeployerPod, deployment.GetName())} + mcPod := []string{fmt.Sprintf(MonitoringConsolePod, deployment.GetName())} + cmAppSourceInfo := AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} + shcAppSourceInfo := AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} + mcAppSourceInfo := AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} + allAppSourceInfo := []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} + ClusterMasterBundleHash := AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + + // Downgrade apps + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) + backend.DeleteFiles(ctx, *uploadedApps) + *uploadedApps = nil + + resourceVersion = GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) + + appVersion = "V1" + appFileList = GetAppFileList(appListV1) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexers", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexers", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + + ClusterMasterReady(ctx, deployment, testcaseEnvInst) + SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) + VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + splunkPodAge = GetPodsStartTime(testcaseEnvInst.GetName()) + + // Final verifications + cmAppSourceInfo.CrAppVersion = appVersion + cmAppSourceInfo.CrAppList = appListV1 + cmAppSourceInfo.CrAppFileList = GetAppFileList(appListV1) + shcAppSourceInfo.CrAppVersion = appVersion + shcAppSourceInfo.CrAppList = appListV1 + shcAppSourceInfo.CrAppFileList = GetAppFileList(appListV1) + mcAppSourceInfo.CrAppVersion = appVersion + mcAppSourceInfo.CrAppList = appListV1 + mcAppSourceInfo.CrAppFileList = GetAppFileList(appListV1) + allAppSourceInfo = []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} + AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) + + VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) +} + +// C3AppFrameworkLocalScopeUpgradeTest deploys a C3 SVA with App Framework +// enabled using local scope, installs V1 apps, then upgrades them to V2. +func C3AppFrameworkLocalScopeUpgradeTest( + ctx context.Context, + backend CloudStorageBackend, + testcaseEnvInst *TestCaseEnv, + deployment *Deployment, + appListV1, appListV2 []string, + downloadDirV1, downloadDirV2 string, + uploadedApps *[]string, +) { + appVersion := "V1" + appFileList := GetAppFileList(appListV1) + + // Upload V1 apps for Indexer Cluster + testDirIdxc := "c3appfw-idxc-" + RandomDNSName(4) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + // Upload V1 apps for Search Head Cluster + testDirShc := "c3appfw-shc-" + RandomDNSName(4) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeLocal + RandomDNSName(3) + appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeLocal + RandomDNSName(3) + appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + RandomDNSName(3) + appSourceVolumeNameShc := "appframework-test-volume-shc-" + RandomDNSName(3) + appFrameworkSpecIdxc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) + + // Deploy C3 CRD + indexerReplicas := 3 + shReplicas := 3 + testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") + cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") + + ClusterMasterReady(ctx, deployment, testcaseEnvInst) + SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + + splunkPodAge := GetPodsStartTime(testcaseEnvInst.GetName()) + + // Initial verifications + idxcPodNames := GeneratePodNameSlice(IndexerPod, deployment.GetName(), indexerReplicas, false, 1) + shcPodNames := GeneratePodNameSlice(SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) + cmPod := []string{fmt.Sprintf(ClusterMasterPod, deployment.GetName())} + deployerPod := []string{fmt.Sprintf(DeployerPod, deployment.GetName())} + cmAppSourceInfo := AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} + shcAppSourceInfo := AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} + allAppSourceInfo := []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} + AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + + // Upgrade apps + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) + backend.DeleteFiles(ctx, *uploadedApps) + *uploadedApps = nil + + appVersion = "V2" + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps", appVersion)) + appFileList = GetAppFileList(appListV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + + ClusterMasterReady(ctx, deployment, testcaseEnvInst) + SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + + splunkPodAge = GetPodsStartTime(testcaseEnvInst.GetName()) + + // Upgrade verifications + cmAppSourceInfo.CrAppVersion = appVersion + cmAppSourceInfo.CrAppList = appListV2 + cmAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) + shcAppSourceInfo.CrAppVersion = appVersion + shcAppSourceInfo.CrAppList = appListV2 + shcAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) + allAppSourceInfo = []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} + AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) +} From ffa2eda6c870bee68eb7ac92160b1e293cd33c42 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Wed, 11 Mar 2026 10:02:52 +0100 Subject: [PATCH 03/12] Single test case + unify cluster rovider --- ...AL2023-build-test-push-workflow-AL2023.yml | 2 +- .../arm-AL2023-int-test-workflow.yml | 2 +- .../arm-RHEL-build-test-push-workflow.yml | 2 +- .../workflows/arm-RHEL-int-test-workflow.yml | 2 +- .../arm-Ubuntu-build-test-push-workflow.yml | 2 +- .../arm-Ubuntu-int-test-workflow.yml | 2 +- .../workflows/build-test-push-workflow.yml | 2 +- .../distroless-build-test-push-workflow.yml | 2 +- .../distroless-int-test-workflow.yml | 2 +- .github/workflows/helm-test-workflow.yml | 2 +- .github/workflows/int-test-azure-workflow.yml | 2 - .github/workflows/int-test-gcp-workflow.yml | 1 - .github/workflows/int-test-workflow.yml | 2 +- .../workflows/manual-int-test-workflow.yml | 2 +- .../namespace-scope-int-workflow.yml | 2 +- .../workflows/nightly-int-test-workflow.yml | 2 +- AGENTS.md | 4 +- .../c3/appframework_suite_test.go} | 54 +- .../c3/appframework_test.go} | 380 +- test/appframework/c3/c3_testflows_test.go | 413 ++ .../c3/manager_appframework_test.go | 428 +-- .../m4/appframework_suite_test.go} | 67 +- .../m4/appframework_test.go} | 334 +- .../m4/manager_appframework_test.go | 334 +- .../s1/appframework_suite_test.go} | 68 +- .../s1/appframework_test.go} | 186 +- .../c3/appframework_azure_suite_test.go | 101 - .../c3/appframework_azure_test.go | 2857 -------------- .../c3/manager_appframework_azure_test.go | 3268 ---------------- .../m4/appframework_azure_suite_test.go | 105 - .../m4/appframework_azure_test.go | 2691 ------------- .../m4/manager_appframework_azure_test.go | 2690 ------------- .../s1/appframework_azure_suite_test.go | 101 - .../s1/appframework_azure_test.go | 2016 ---------- .../c3/appframework_gcs_suite_test.go | 102 - .../c3/appframework_gcs_test.go | 170 - .../c3/manager_appframework_test.go | 3420 ----------------- .../m4/appframework_gcs_suite_test.go | 103 - .../m4/appframework_gcs_test.go | 2709 ------------- .../m4/manager_appframework_test.go | 2708 ------------- .../s1/appframework_gcs_suite_test.go | 98 - .../s1/appframework_gcs_test.go | 2055 ---------- test/deploy-cluster.sh | 8 +- test/env.sh | 1 - test/testenv/appframework_c3_testflows.go | 412 -- test/testenv/cloud_storage_backend.go | 3 +- test/testenv/s3utils.go | 2 +- 47 files changed, 1369 insertions(+), 26550 deletions(-) rename test/{appframework_aws/c3/appframework_aws_suite_test.go => appframework/c3/appframework_suite_test.go} (64%) rename test/{appframework_aws/c3/appframework_aws_test.go => appframework/c3/appframework_test.go} (91%) create mode 100644 test/appframework/c3/c3_testflows_test.go rename test/{appframework_aws => appframework}/c3/manager_appframework_test.go (92%) rename test/{appframework_aws/m4/appframework_aws_suite_test.go => appframework/m4/appframework_suite_test.go} (62%) rename test/{appframework_aws/m4/appframework_aws_test.go => appframework/m4/appframework_test.go} (93%) rename test/{appframework_aws => appframework}/m4/manager_appframework_test.go (93%) rename test/{appframework_aws/s1/appframework_aws_suite_test.go => appframework/s1/appframework_suite_test.go} (59%) rename test/{appframework_aws/s1/appframework_aws_test.go => appframework/s1/appframework_test.go} (93%) delete mode 100644 test/appframework_az/c3/appframework_azure_suite_test.go delete mode 100644 test/appframework_az/c3/appframework_azure_test.go delete mode 100644 test/appframework_az/c3/manager_appframework_azure_test.go delete mode 100644 test/appframework_az/m4/appframework_azure_suite_test.go delete mode 100644 test/appframework_az/m4/appframework_azure_test.go delete mode 100644 test/appframework_az/m4/manager_appframework_azure_test.go delete mode 100644 test/appframework_az/s1/appframework_azure_suite_test.go delete mode 100644 test/appframework_az/s1/appframework_azure_test.go delete mode 100644 test/appframework_gcp/c3/appframework_gcs_suite_test.go delete mode 100644 test/appframework_gcp/c3/appframework_gcs_test.go delete mode 100644 test/appframework_gcp/c3/manager_appframework_test.go delete mode 100644 test/appframework_gcp/m4/appframework_gcs_suite_test.go delete mode 100644 test/appframework_gcp/m4/appframework_gcs_test.go delete mode 100644 test/appframework_gcp/m4/manager_appframework_test.go delete mode 100644 test/appframework_gcp/s1/appframework_gcs_suite_test.go delete mode 100644 test/appframework_gcp/s1/appframework_gcs_test.go delete mode 100644 test/testenv/appframework_c3_testflows.go diff --git a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml index 143111628..4a00c2a4d 100644 --- a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml +++ b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml @@ -149,7 +149,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/arm-AL2023-int-test-workflow.yml b/.github/workflows/arm-AL2023-int-test-workflow.yml index 39986ebca..3e1949623 100644 --- a/.github/workflows/arm-AL2023-int-test-workflow.yml +++ b/.github/workflows/arm-AL2023-int-test-workflow.yml @@ -83,7 +83,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/arm-RHEL-build-test-push-workflow.yml b/.github/workflows/arm-RHEL-build-test-push-workflow.yml index 07848683c..ff9de5ef6 100644 --- a/.github/workflows/arm-RHEL-build-test-push-workflow.yml +++ b/.github/workflows/arm-RHEL-build-test-push-workflow.yml @@ -85,7 +85,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/arm-RHEL-int-test-workflow.yml b/.github/workflows/arm-RHEL-int-test-workflow.yml index 83c7e8fb9..0884ff0d7 100644 --- a/.github/workflows/arm-RHEL-int-test-workflow.yml +++ b/.github/workflows/arm-RHEL-int-test-workflow.yml @@ -83,7 +83,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml index 05aa9a1fc..cf495ee48 100644 --- a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml +++ b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml @@ -149,7 +149,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/arm-Ubuntu-int-test-workflow.yml b/.github/workflows/arm-Ubuntu-int-test-workflow.yml index b3f133abe..f73b8562b 100644 --- a/.github/workflows/arm-Ubuntu-int-test-workflow.yml +++ b/.github/workflows/arm-Ubuntu-int-test-workflow.yml @@ -83,7 +83,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index e457272b8..3d6396b84 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -213,7 +213,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/distroless-build-test-push-workflow.yml b/.github/workflows/distroless-build-test-push-workflow.yml index 69a005c8d..1dc14da9c 100644 --- a/.github/workflows/distroless-build-test-push-workflow.yml +++ b/.github/workflows/distroless-build-test-push-workflow.yml @@ -208,7 +208,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing smoke keyword TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/distroless-int-test-workflow.yml b/.github/workflows/distroless-int-test-workflow.yml index 01a024247..604ce5064 100644 --- a/.github/workflows/distroless-int-test-workflow.yml +++ b/.github/workflows/distroless-int-test-workflow.yml @@ -87,7 +87,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/helm-test-workflow.yml b/.github/workflows/helm-test-workflow.yml index 7aceea88a..40ceade3a 100644 --- a/.github/workflows/helm-test-workflow.yml +++ b/.github/workflows/helm-test-workflow.yml @@ -64,7 +64,7 @@ jobs: SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/int-test-azure-workflow.yml b/.github/workflows/int-test-azure-workflow.yml index 743bb5df6..172983ed4 100644 --- a/.github/workflows/int-test-azure-workflow.yml +++ b/.github/workflows/int-test-azure-workflow.yml @@ -52,7 +52,6 @@ jobs: needs: build-operator-image env: CLUSTER_WORKERS: 5 - TEST_CLUSTER_PLATFORM: azure CLUSTER_PROVIDER: azure AZURE_CONTAINER_REGISTRY: ${{secrets.AZURE_CONTAINER_REGISTRY}} CLUSTER_NAME: ${{secrets.CLUSTER_NAME}} @@ -120,7 +119,6 @@ jobs: TEST_FOCUS: azure_sanity # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: azure CLUSTER_PROVIDER: azure AZURE_CONTAINER_REGISTRY: ${{secrets.AZURE_CONTAINER_REGISTRY}} CLUSTER_NAME: ${{secrets.CLUSTER_NAME}} diff --git a/.github/workflows/int-test-gcp-workflow.yml b/.github/workflows/int-test-gcp-workflow.yml index c72c05579..24a266de7 100644 --- a/.github/workflows/int-test-gcp-workflow.yml +++ b/.github/workflows/int-test-gcp-workflow.yml @@ -79,7 +79,6 @@ jobs: needs: build-operator-image env: CLUSTER_WORKERS: 5 - TEST_CLUSTER_PLATFORM: gcp CLUSTER_PROVIDER: gcp ARTIFACT_REGISTRY: ${{ secrets.GCP_ARTIFACT_REGISTRY }} GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 001a34cee..4a25190ba 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -83,7 +83,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/manual-int-test-workflow.yml b/.github/workflows/manual-int-test-workflow.yml index 96629316e..1beba528c 100644 --- a/.github/workflows/manual-int-test-workflow.yml +++ b/.github/workflows/manual-int-test-workflow.yml @@ -42,7 +42,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/namespace-scope-int-workflow.yml b/.github/workflows/namespace-scope-int-workflow.yml index fc3d3554f..39e29e058 100644 --- a/.github/workflows/namespace-scope-int-workflow.yml +++ b/.github/workflows/namespace-scope-int-workflow.yml @@ -37,7 +37,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml index 3f886dd9a..7f07afdc4 100644 --- a/.github/workflows/nightly-int-test-workflow.yml +++ b/.github/workflows/nightly-int-test-workflow.yml @@ -79,7 +79,7 @@ jobs: TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" - TEST_CLUSTER_PLATFORM: eks + CLUSTER_PROVIDER: eks EKS_VPC_PRIVATE_SUBNET_STRING: ${{ secrets.EKS_VPC_PRIVATE_SUBNET_STRING }} EKS_VPC_PUBLIC_SUBNET_STRING: ${{ secrets.EKS_VPC_PUBLIC_SUBNET_STRING }} TEST_BUCKET: ${{ secrets.TEST_BUCKET }} diff --git a/AGENTS.md b/AGENTS.md index e2ba5781f..015ebe9d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -152,9 +152,7 @@ Test coverage includes: - `test/smoke/` - Basic smoke tests - `test/licensemanager/` - License manager tests - `test/monitoring_console/` - Monitoring console tests -- `test/appframework_aws/` - App Framework with AWS S3 -- `test/appframework_az/` - App Framework with Azure Blob -- `test/appframework_gcp/` - App Framework with GCP Storage +- `test/appframework/` - App Framework tests (provider-agnostic, runs on AWS/Azure/GCP) - `test/smartstore/` - SmartStore functionality - `test/secret/` - Secret management - `test/custom_resource_crud/` - CR CRUD operations diff --git a/test/appframework_aws/c3/appframework_aws_suite_test.go b/test/appframework/c3/appframework_suite_test.go similarity index 64% rename from test/appframework_aws/c3/appframework_aws_suite_test.go rename to test/appframework/c3/appframework_suite_test.go index aa1dde42d..86242b8db 100644 --- a/test/appframework_aws/c3/appframework_aws_suite_test.go +++ b/test/appframework/c3/appframework_suite_test.go @@ -14,6 +14,7 @@ package c3appfw import ( + "context" "os" "path/filepath" "testing" @@ -40,21 +41,21 @@ var ( testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) appListV1 []string appListV2 []string - testDataS3Bucket = os.Getenv("TEST_BUCKET") - testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation + testDataBucket = os.Getenv("TEST_BUCKET") + testBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") + appDirV1 = testenv.AppLocationV1 + appDirV2 = testenv.AppLocationV2 + pvTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) downloadDirPVTestApps = filepath.Join(currDir, "c3appfwPVTestApps-"+testenv.RandomDNSName(4)) + backend testenv.CloudStorageBackend ) // TestBasic is the main entry point func TestBasic(t *testing.T) { - // Find and load the .env file from the current directory upwards if err := loadEnvFile(); err != nil { panic("Error loading .env file: " + err.Error()) } @@ -66,32 +67,21 @@ func TestBasic(t *testing.T) { RunSpecs(t, "Running "+testSuiteName, sc) } -//func TestMain(m *testing.M) { -// Run the tests -// os.Exit(m.Run()) -//} - // loadEnvFile traverses up the directory tree to find a .env file func loadEnvFile() error { - // Get the current working directory dir, err := os.Getwd() if err != nil { return err } - // Traverse up the directory tree for { - // Check if .env file exists in the current directory envFile := filepath.Join(dir, ".env") if _, err := os.Stat(envFile); err == nil { - // .env file found, load it return godotenv.Load(envFile) } - // Move up to the parent directory parentDir := filepath.Dir(dir) if parentDir == dir { - // Reached the root directory return nil } dir = parentDir @@ -99,30 +89,24 @@ func loadEnvFile() error { } var _ = BeforeSuite(func() { + ctx := context.TODO() var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) Expect(err).ToNot(HaveOccurred()) - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) + backend = testenv.NewCloudStorageBackend(testBucket, testDataBucket) - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") + appListV1 = testenv.BasicApps + appFileList := testenv.GetAppFileList(appListV1) - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) + appFileList = testenv.GetAppFileList(appListV2) + err = backend.DownloadFiles(ctx, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") }) var _ = AfterSuite(func() { @@ -130,10 +114,6 @@ var _ = AfterSuite(func() { Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) } - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - // Delete locally downloaded app files err := os.RemoveAll(downloadDirV1) Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework/c3/appframework_test.go similarity index 91% rename from test/appframework_aws/c3/appframework_aws_test.go rename to test/appframework/c3/appframework_test.go index 5b29a7eb9..b8a7824d5 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework/c3/appframework_test.go @@ -39,12 +39,12 @@ var _ = Describe("c3appfw test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var s3TestDirShc string - var s3TestDirIdxc string - var s3TestDirShcLocal string - var s3TestDirIdxcLocal string - var s3TestDirShcCluster string - var s3TestDirIdxcCluster string + var testDirShc string + var testDirIdxc string + var testDirShcLocal string + var testDirIdxcLocal string + var testDirShcCluster string + var testDirIdxcCluster string var appSourceNameIdxc string var appSourceNameShc string var uploadedApps []string @@ -60,7 +60,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) Expect(err).To(Succeed(), "Unable to create testcaseenv") testenv.SpecifiedTestTimeout = 4000 - backend = testenv.NewCloudStorageBackend(testS3Bucket, testDataS3Bucket) + backend = testenv.NewCloudStorageBackend(testBucket, testDataBucket) deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") }) @@ -93,13 +93,13 @@ var _ = Describe("c3appfw test", func() { Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { It("smoke, c3, masterappframeworkc3, appframework: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - testenv.C3AppFrameworkUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) + C3AppFrameworkUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { It("smoke, c3, masterappframeworkc3, appframework: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - testenv.C3AppFrameworkDowngradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) + C3AppFrameworkDowngradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) }) }) @@ -147,15 +147,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -164,8 +164,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -411,17 +411,17 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -430,8 +430,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -473,17 +473,17 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -559,32 +559,32 @@ var _ = Describe("c3appfw test", func() { // Upload appListLocal list of apps to S3 (to be used for local install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) + testDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + testDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + testDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + testDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -600,7 +600,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -608,18 +608,18 @@ var _ = Describe("c3appfw test", func() { VolName: appSourceVolumeNameIdxcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, testDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, testDirShcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -664,7 +664,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -675,19 +675,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -769,31 +769,31 @@ var _ = Describe("c3appfw test", func() { // Upload appListLocal list of apps to S3 (to be used for local install) for Idxc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) + testDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + testDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + testDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + testDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -808,25 +808,25 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, testDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, testDirShcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -872,7 +872,7 @@ var _ = Describe("c3appfw test", func() { //############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -883,19 +883,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -965,7 +965,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -973,14 +973,14 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appList) // Upload app to S3 for Indexer Cluster - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload app to S3 for Search Head Cluster - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -989,8 +989,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1070,15 +1070,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1101,14 +1101,14 @@ var _ = Describe("c3appfw test", func() { testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) // Upload V1 apps to S3 for Indexer Cluster - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1117,8 +1117,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer indexerReplicas := 3 @@ -1164,7 +1164,7 @@ var _ = Describe("c3appfw test", func() { // ############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for C3 @@ -1172,17 +1172,17 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to S3", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1310,17 +1310,17 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1329,8 +1329,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1371,17 +1371,17 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1497,31 +1497,31 @@ var _ = Describe("c3appfw test", func() { // Upload appListLocal list of apps to S3 (to be used for local install) for Idxc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) + testDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + testDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + testDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + testDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1536,25 +1536,25 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, testDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, testDirShcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -1598,7 +1598,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1609,19 +1609,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1706,15 +1706,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1739,22 +1739,22 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1763,8 +1763,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -1779,13 +1779,13 @@ var _ = Describe("c3appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1835,15 +1835,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1868,22 +1868,22 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1892,8 +1892,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -1908,13 +1908,13 @@ var _ = Describe("c3appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1971,21 +1971,21 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1994,8 +1994,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2065,21 +2065,21 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2088,8 +2088,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2162,15 +2162,15 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2179,8 +2179,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2220,7 +2220,7 @@ var _ = Describe("c3appfw test", func() { testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) // Disable the app - testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) + testenv.DisableAppsToS3(downloadDirV1, appFileName, testDirIdxc) // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) @@ -2235,8 +2235,8 @@ var _ = Describe("c3appfw test", func() { testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) // Delete the file from S3 - s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + s3Filepath := filepath.Join(testDirIdxc, appFileName[0]) + err = testenv.DeleteFileOnS3(testBucket, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -2277,20 +2277,20 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2299,8 +2299,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 120) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2317,13 +2317,13 @@ var _ = Describe("c3appfw test", func() { appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2394,20 +2394,20 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = backend.DownloadFiles(ctx, pvTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) + testDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) - s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) + testDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2419,9 +2419,9 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -2483,21 +2483,21 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2506,8 +2506,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2575,15 +2575,15 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2592,8 +2592,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2707,36 +2707,36 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) // Upload ES app to S3 testcaseEnvInst.Log.Info("Upload ES app to S3") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload Technology add-on apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileListIdxc, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileListIdxc, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, testDirShc, 180) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -2747,7 +2747,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 180) // Deploy C3 SVA // Deploy the Cluster Master @@ -2822,12 +2822,12 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = backend.DownloadFiles(ctx, appDirV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to S3 for Search Head Cluster", appVersion)) - // uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + // uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to S3 test directory for Search Head Cluster", appVersion)) // uploadedApps = append(uploadedApps, uploadedFiles...) diff --git a/test/appframework/c3/c3_testflows_test.go b/test/appframework/c3/c3_testflows_test.go new file mode 100644 index 000000000..516d88532 --- /dev/null +++ b/test/appframework/c3/c3_testflows_test.go @@ -0,0 +1,413 @@ +package c3appfw + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/splunk/enterprise" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" +) + +// C3AppFrameworkUpgradeTest deploys a C3 SVA with App Framework enabled, +// installs V1 apps (with MC), then upgrades them to V2 and re-verifies. +// This is the shared flow behind the "upgrade" smoke test in every cloud provider. +func C3AppFrameworkUpgradeTest( + ctx context.Context, + backend testenv.CloudStorageBackend, + testcaseEnvInst *testenv.TestCaseEnv, + deployment *testenv.Deployment, + appListV1, appListV2 []string, + downloadDirV1, downloadDirV2 string, + uploadedApps *[]string, +) { + appVersion := "V1" + appFileList := testenv.GetAppFileList(appListV1) + + // Upload V1 apps for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) + appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) + + mcSpec := enterpriseApi.MonitoringConsoleSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + AppFrameworkConfig: appFrameworkSpecMC, + } + + testcaseEnvInst.Log.Info("Deploy Monitoring Console") + mcName := deployment.GetName() + mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + // Upload V1 apps for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) + testDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + // Upload V1 apps for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + testDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) + appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) + appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) + appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) + + resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) + + testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") + indexerReplicas := 3 + shReplicas := 3 + cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") + + testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) + + splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") + ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) + _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) + Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) + scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} + allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) + testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + + // Initial verifications + idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) + shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) + cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} + deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} + mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} + cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} + shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} + mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} + allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} + ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + + // Upgrade apps + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) + backend.DeleteFiles(ctx, *uploadedApps) + *uploadedApps = nil + + resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) + + appVersion = "V2" + appFileList = testenv.GetAppFileList(appListV2) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + + testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + // Final verifications + cmAppSourceInfo.CrAppVersion = appVersion + cmAppSourceInfo.CrAppList = appListV2 + cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) + shcAppSourceInfo.CrAppVersion = appVersion + shcAppSourceInfo.CrAppList = appListV2 + shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) + mcAppSourceInfo.CrAppVersion = appVersion + mcAppSourceInfo.CrAppList = appListV2 + mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) + allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} + testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) + + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) +} + +// C3AppFrameworkDowngradeTest deploys a C3 SVA with App Framework enabled, +// installs V2 apps (with MC), then downgrades them to V1 and re-verifies. +func C3AppFrameworkDowngradeTest( + ctx context.Context, + backend testenv.CloudStorageBackend, + testcaseEnvInst *testenv.TestCaseEnv, + deployment *testenv.Deployment, + appListV1, appListV2 []string, + downloadDirV1, downloadDirV2 string, + uploadedApps *[]string, +) { + appVersion := "V2" + appFileList := testenv.GetAppFileList(appListV2) + + // Upload V2 apps for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) + appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) + + mcSpec := enterpriseApi.MonitoringConsoleSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + AppFrameworkConfig: appFrameworkSpecMC, + } + + testcaseEnvInst.Log.Info("Deploy Monitoring Console") + mcName := deployment.GetName() + mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + // Upload V2 apps for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) + testDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + // Upload V2 apps for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + testDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) + appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) + appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) + appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) + + resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) + + testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") + indexerReplicas := 3 + shReplicas := 3 + cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") + + testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + // Initial verifications + idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) + shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) + cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} + deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} + mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} + cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} + shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} + mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} + allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} + ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + + // Downgrade apps + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) + backend.DeleteFiles(ctx, *uploadedApps) + *uploadedApps = nil + + resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) + + appVersion = "V1" + appFileList = testenv.GetAppFileList(appListV1) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexers", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexers", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + + testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + // Final verifications + cmAppSourceInfo.CrAppVersion = appVersion + cmAppSourceInfo.CrAppList = appListV1 + cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) + shcAppSourceInfo.CrAppVersion = appVersion + shcAppSourceInfo.CrAppList = appListV1 + shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) + mcAppSourceInfo.CrAppVersion = appVersion + mcAppSourceInfo.CrAppList = appListV1 + mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) + allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} + testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) + + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) +} + +// C3AppFrameworkLocalScopeUpgradeTest deploys a C3 SVA with App Framework +// enabled using local scope, installs V1 apps, then upgrades them to V2. +func C3AppFrameworkLocalScopeUpgradeTest( + ctx context.Context, + backend testenv.CloudStorageBackend, + testcaseEnvInst *testenv.TestCaseEnv, + deployment *testenv.Deployment, + appListV1, appListV2 []string, + downloadDirV1, downloadDirV2 string, + uploadedApps *[]string, +) { + appVersion := "V1" + appFileList := testenv.GetAppFileList(appListV1) + + // Upload V1 apps for Indexer Cluster + testDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + // Upload V1 apps for Search Head Cluster + testDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) + appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) + appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) + appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) + + // Deploy C3 CRD + indexerReplicas := 3 + shReplicas := 3 + testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") + cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") + + testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + + splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + // Initial verifications + idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) + shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) + cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} + deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} + cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} + shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} + allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} + testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + + // Upgrade apps + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) + backend.DeleteFiles(ctx, *uploadedApps) + *uploadedApps = nil + + appVersion = "V2" + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps", appVersion)) + appFileList = testenv.GetAppFileList(appListV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) + *uploadedApps = append(*uploadedApps, uploadedFiles...) + + testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + + testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) + + splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + // Upgrade verifications + cmAppSourceInfo.CrAppVersion = appVersion + cmAppSourceInfo.CrAppList = appListV2 + cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) + shcAppSourceInfo.CrAppVersion = appVersion + shcAppSourceInfo.CrAppList = appListV2 + shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) + allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} + testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") + + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) +} diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework/c3/manager_appframework_test.go similarity index 92% rename from test/appframework_aws/c3/manager_appframework_test.go rename to test/appframework/c3/manager_appframework_test.go index ba0b3e8ea..7d63be517 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework/c3/manager_appframework_test.go @@ -37,12 +37,12 @@ var _ = Describe("c3appfw test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var s3TestDirShc string - var s3TestDirIdxc string - var s3TestDirShcLocal string - var s3TestDirIdxcLocal string - var s3TestDirShcCluster string - var s3TestDirIdxcCluster string + var testDirShc string + var testDirIdxc string + var testDirShcLocal string + var testDirIdxcLocal string + var testDirShcCluster string + var testDirIdxcCluster string var appSourceNameIdxc string var appSourceNameShc string var uploadedApps []string @@ -75,7 +75,7 @@ var _ = Describe("c3appfw test", func() { // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) } if filePresentOnOperator { @@ -123,15 +123,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -155,15 +155,15 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -172,8 +172,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // get revision number of the resource resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) @@ -238,7 +238,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -248,19 +248,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -484,15 +484,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V2" appFileList := testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ @@ -517,15 +517,15 @@ var _ = Describe("c3appfw test", func() { // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -534,8 +534,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // get revision number of the resource resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) @@ -588,7 +588,7 @@ var _ = Describe("c3appfw test", func() { //############## DOWNGRADE APPS ############### // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -598,19 +598,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -699,15 +699,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -716,8 +716,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -963,17 +963,17 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -982,8 +982,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD indexerReplicas := 3 @@ -1025,17 +1025,17 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1111,32 +1111,32 @@ var _ = Describe("c3appfw test", func() { // Upload appListLocal list of apps to S3 (to be used for local install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) + testDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + testDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + testDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + testDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1152,7 +1152,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1160,18 +1160,18 @@ var _ = Describe("c3appfw test", func() { VolName: appSourceVolumeNameIdxcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, testDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, testDirShcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -1216,7 +1216,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1227,19 +1227,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1321,31 +1321,31 @@ var _ = Describe("c3appfw test", func() { // Upload appListLocal list of apps to S3 (to be used for local install) for Idxc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) + testDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + testDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + testDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + testDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1360,25 +1360,25 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, testDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, testDirShcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -1424,7 +1424,7 @@ var _ = Describe("c3appfw test", func() { //############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1435,19 +1435,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1517,7 +1517,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1525,14 +1525,14 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appList) // Upload app to S3 for Indexer Cluster - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload app to S3 for Search Head Cluster - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1541,8 +1541,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") @@ -1622,15 +1622,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1653,14 +1653,14 @@ var _ = Describe("c3appfw test", func() { testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) // Upload V1 apps to S3 for Indexer Cluster - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1669,8 +1669,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 0) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 @@ -1716,7 +1716,7 @@ var _ = Describe("c3appfw test", func() { // ############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for C3 @@ -1724,17 +1724,17 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to S3", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1862,17 +1862,17 @@ var _ = Describe("c3appfw test", func() { //################## SETUP #################### // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1881,8 +1881,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 0) // Deploy C3 CRD indexerReplicas := 3 @@ -1923,17 +1923,17 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2049,31 +2049,31 @@ var _ = Describe("c3appfw test", func() { // Upload appListLocal list of apps to S3 (to be used for local install) for Idxc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) + testDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + testDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + testDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + testDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2088,25 +2088,25 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, testDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, testDirShcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2150,7 +2150,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -2161,19 +2161,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2258,15 +2258,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2291,22 +2291,22 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2315,8 +2315,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2331,13 +2331,13 @@ var _ = Describe("c3appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2387,15 +2387,15 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -2420,22 +2420,22 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2444,8 +2444,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2460,13 +2460,13 @@ var _ = Describe("c3appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2523,21 +2523,21 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2546,8 +2546,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2617,21 +2617,21 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2640,8 +2640,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2714,15 +2714,15 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2731,8 +2731,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2772,7 +2772,7 @@ var _ = Describe("c3appfw test", func() { testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) // Disable the app - testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) + testenv.DisableAppsToS3(downloadDirV1, appFileName, testDirIdxc) // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) @@ -2787,8 +2787,8 @@ var _ = Describe("c3appfw test", func() { testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) // Delete the file from S3 - s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + s3Filepath := filepath.Join(testDirIdxc, appFileName[0]) + err = testenv.DeleteFileOnS3(testBucket, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -2829,20 +2829,20 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2851,8 +2851,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 120) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 120) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -2869,13 +2869,13 @@ var _ = Describe("c3appfw test", func() { appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2946,20 +2946,20 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = backend.DownloadFiles(ctx, pvTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) + testDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) - s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) + testDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2971,9 +2971,9 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy C3 CRD @@ -3035,21 +3035,21 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3058,8 +3058,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3127,15 +3127,15 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3144,8 +3144,8 @@ var _ = Describe("c3appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy C3 CRD testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") @@ -3259,36 +3259,36 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) + testDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + testDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) // Upload ES app to S3 testcaseEnvInst.Log.Info("Upload ES app to S3") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload Technology add-on apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileListIdxc, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileListIdxc, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, testDirShc, 180) appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -3299,7 +3299,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 180) // Deploy C3 SVA // Deploy the Cluster Manager @@ -3374,12 +3374,12 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = backend.DownloadFiles(ctx, appDirV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to S3 for Search Head Cluster", appVersion)) - // uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + // uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to S3 test directory for Search Head Cluster", appVersion)) // uploadedApps = append(uploadedApps, uploadedFiles...) diff --git a/test/appframework_aws/m4/appframework_aws_suite_test.go b/test/appframework/m4/appframework_suite_test.go similarity index 62% rename from test/appframework_aws/m4/appframework_aws_suite_test.go rename to test/appframework/m4/appframework_suite_test.go index aa21c7084..dfe7c74b3 100644 --- a/test/appframework_aws/m4/appframework_aws_suite_test.go +++ b/test/appframework/m4/appframework_suite_test.go @@ -14,6 +14,7 @@ package m4appfw import ( + "context" "os" "path/filepath" "testing" @@ -22,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/joho/godotenv" "github.com/splunk/splunk-operator/test/testenv" ) @@ -39,20 +41,24 @@ var ( testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) appListV1 []string appListV2 []string - testDataS3Bucket = os.Getenv("TEST_BUCKET") - testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation + testDataBucket = os.Getenv("TEST_BUCKET") + testBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") + appDirV1 = testenv.AppLocationV1 + appDirV2 = testenv.AppLocationV2 + pvTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) downloadDirPVTestApps = filepath.Join(currDir, "m4appfwPVTestApps-"+testenv.RandomDNSName(4)) + backend testenv.CloudStorageBackend ) // TestBasic is the main entry point func TestBasic(t *testing.T) { + if err := loadEnvFile(); err != nil { + panic("Error loading .env file: " + err.Error()) + } RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -61,31 +67,46 @@ func TestBasic(t *testing.T) { RunSpecs(t, "Running "+testSuiteName, sc) } +// loadEnvFile traverses up the directory tree to find a .env file +func loadEnvFile() error { + dir, err := os.Getwd() + if err != nil { + return err + } + + for { + envFile := filepath.Join(dir, ".env") + if _, err := os.Stat(envFile); err == nil { + return godotenv.Load(envFile) + } + + parentDir := filepath.Dir(dir) + if parentDir == dir { + return nil + } + dir = parentDir + } +} + var _ = BeforeSuite(func() { + ctx := context.TODO() var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) Expect(err).ToNot(HaveOccurred()) - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) + backend = testenv.NewCloudStorageBackend(testBucket, testDataBucket) - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") + appListV1 = testenv.BasicApps + appFileList := testenv.GetAppFileList(appListV1) - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) + appFileList = testenv.GetAppFileList(appListV2) + err = backend.DownloadFiles(ctx, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") }) var _ = AfterSuite(func() { @@ -93,10 +114,6 @@ var _ = AfterSuite(func() { Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) } - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - // Delete locally downloaded app files err := os.RemoveAll(downloadDirV1) Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") diff --git a/test/appframework_aws/m4/appframework_aws_test.go b/test/appframework/m4/appframework_test.go similarity index 93% rename from test/appframework_aws/m4/appframework_aws_test.go rename to test/appframework/m4/appframework_test.go index eab0b7023..3c96bee17 100644 --- a/test/appframework_aws/m4/appframework_aws_test.go +++ b/test/appframework/m4/appframework_test.go @@ -40,14 +40,14 @@ var _ = Describe("m4appfw test", func() { var uploadedApps []string var appSourceNameIdxc string var appSourceNameShc string - var s3TestDirShc string - var s3TestDirIdxc string + var testDirShc string + var testDirIdxc string var appSourceVolumeNameIdxc string var appSourceVolumeNameShc string - var s3TestDirShcLocal string - var s3TestDirIdxcLocal string - var s3TestDirShcCluster string - var s3TestDirIdxcCluster string + var testDirShcLocal string + var testDirIdxcLocal string + var testDirShcCluster string + var testDirIdxcCluster string var filePresentOnOperator bool ctx := context.TODO() @@ -59,8 +59,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to create testcaseenv") deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) + testDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) + testDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) }) @@ -75,7 +75,7 @@ var _ = Describe("m4appfw test", func() { } // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) @@ -125,15 +125,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -156,21 +156,21 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // get revision number of the resource resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) @@ -236,7 +236,7 @@ var _ = Describe("m4appfw test", func() { //############# UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -246,19 +246,19 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -344,15 +344,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V2" appFileList := testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -375,21 +375,21 @@ var _ = Describe("m4appfw test", func() { // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -441,26 +441,26 @@ var _ = Describe("m4appfw test", func() { // Delete V2 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V1 apps to S3 for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -552,21 +552,21 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -818,21 +818,21 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -872,20 +872,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -958,15 +958,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -989,21 +989,21 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 0) siteCount := 3 shReplicas := 3 @@ -1055,19 +1055,19 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1210,21 +1210,21 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -1263,20 +1263,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1395,31 +1395,31 @@ var _ = Describe("m4appfw test", func() { // Upload appListLocal list of apps to S3 (to be used for local install) for Idxc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) + testDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + testDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) + testDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + testDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1435,18 +1435,18 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Cluster master with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, testDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1454,7 +1454,7 @@ var _ = Describe("m4appfw test", func() { VolName: appSourceVolumeNameShcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, testDirShcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -1502,7 +1502,7 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1513,20 +1513,20 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1610,15 +1610,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1643,22 +1643,22 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1667,8 +1667,8 @@ var _ = Describe("m4appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1684,13 +1684,13 @@ var _ = Describe("m4appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1744,15 +1744,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1777,22 +1777,22 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1801,8 +1801,8 @@ var _ = Describe("m4appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1819,13 +1819,13 @@ var _ = Describe("m4appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1881,27 +1881,27 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1973,27 +1973,27 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2069,21 +2069,21 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2127,7 +2127,7 @@ var _ = Describe("m4appfw test", func() { testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) // Disable the app - testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) + testenv.DisableAppsToS3(downloadDirV1, appFileName, testDirIdxc) // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) @@ -2145,8 +2145,8 @@ var _ = Describe("m4appfw test", func() { testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) // Delete the file from S3 - s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + s3Filepath := filepath.Join(testDirIdxc, appFileName[0]) + err = testenv.DeleteFileOnS3(testBucket, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -2186,21 +2186,21 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer siteCount := 3 @@ -2290,20 +2290,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2367,26 +2367,26 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 120) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2404,13 +2404,13 @@ var _ = Describe("m4appfw test", func() { appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2483,20 +2483,20 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = backend.DownloadFiles(ctx, pvTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + testDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirPVTestApps) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) - s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) + testDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2508,9 +2508,9 @@ var _ = Describe("m4appfw test", func() { appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer @@ -2568,27 +2568,27 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2658,21 +2658,21 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") diff --git a/test/appframework_aws/m4/manager_appframework_test.go b/test/appframework/m4/manager_appframework_test.go similarity index 93% rename from test/appframework_aws/m4/manager_appframework_test.go rename to test/appframework/m4/manager_appframework_test.go index d99712693..1bbba550b 100644 --- a/test/appframework_aws/m4/manager_appframework_test.go +++ b/test/appframework/m4/manager_appframework_test.go @@ -39,14 +39,14 @@ var _ = Describe("m4appfw test", func() { var uploadedApps []string var appSourceNameIdxc string var appSourceNameShc string - var s3TestDirShc string - var s3TestDirIdxc string + var testDirShc string + var testDirIdxc string var appSourceVolumeNameIdxc string var appSourceVolumeNameShc string - var s3TestDirShcLocal string - var s3TestDirIdxcLocal string - var s3TestDirShcCluster string - var s3TestDirIdxcCluster string + var testDirShcLocal string + var testDirIdxcLocal string + var testDirShcCluster string + var testDirIdxcCluster string var filePresentOnOperator bool ctx := context.TODO() @@ -58,8 +58,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to create testcaseenv") deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) + testDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) + testDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) }) @@ -74,7 +74,7 @@ var _ = Describe("m4appfw test", func() { } // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) @@ -124,15 +124,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -155,21 +155,21 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // get revision number of the resource resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) @@ -235,7 +235,7 @@ var _ = Describe("m4appfw test", func() { //############# UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -245,19 +245,19 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -343,15 +343,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V2" appFileList := testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -374,21 +374,21 @@ var _ = Describe("m4appfw test", func() { // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -440,26 +440,26 @@ var _ = Describe("m4appfw test", func() { // Delete V2 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V1 apps to S3 for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -551,21 +551,21 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -817,21 +817,21 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -871,20 +871,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -957,15 +957,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -988,21 +988,21 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 0) siteCount := 3 shReplicas := 3 @@ -1054,19 +1054,19 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1209,21 +1209,21 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -1262,20 +1262,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1394,31 +1394,31 @@ var _ = Describe("m4appfw test", func() { // Upload appListLocal list of apps to S3 (to be used for local install) for Idxc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) + testDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) - s3TestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + testDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) + testDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + testDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1434,18 +1434,18 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, testDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1453,7 +1453,7 @@ var _ = Describe("m4appfw test", func() { VolName: appSourceVolumeNameShcCluster, Scope: enterpriseApi.ScopeCluster, } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} + appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, testDirShcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -1501,7 +1501,7 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1512,20 +1512,20 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1609,15 +1609,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1642,22 +1642,22 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1666,8 +1666,8 @@ var _ = Describe("m4appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1683,13 +1683,13 @@ var _ = Describe("m4appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1743,15 +1743,15 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1776,22 +1776,22 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + testDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + testDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1800,8 +1800,8 @@ var _ = Describe("m4appfw test", func() { appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1818,13 +1818,13 @@ var _ = Describe("m4appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1880,27 +1880,27 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -1972,27 +1972,27 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2068,21 +2068,21 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2126,7 +2126,7 @@ var _ = Describe("m4appfw test", func() { testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) // Disable the app - testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) + testenv.DisableAppsToS3(downloadDirV1, appFileName, testDirIdxc) // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) @@ -2144,8 +2144,8 @@ var _ = Describe("m4appfw test", func() { testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) // Delete the file from S3 - s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + s3Filepath := filepath.Join(testDirIdxc, appFileName[0]) + err = testenv.DeleteFileOnS3(testBucket, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -2185,21 +2185,21 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 0) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 0) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 @@ -2289,20 +2289,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2366,26 +2366,26 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 120) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 120) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2403,13 +2403,13 @@ var _ = Describe("m4appfw test", func() { appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2482,20 +2482,20 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = backend.DownloadFiles(ctx, pvTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - s3TestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + testDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirPVTestApps) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) - s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) + testDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2507,9 +2507,9 @@ var _ = Describe("m4appfw test", func() { appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer @@ -2567,27 +2567,27 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") @@ -2657,21 +2657,21 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) + uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) + appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) + appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) // Deploy M4 CRD testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") diff --git a/test/appframework_aws/s1/appframework_aws_suite_test.go b/test/appframework/s1/appframework_suite_test.go similarity index 59% rename from test/appframework_aws/s1/appframework_aws_suite_test.go rename to test/appframework/s1/appframework_suite_test.go index 252889490..7093ff41c 100644 --- a/test/appframework_aws/s1/appframework_aws_suite_test.go +++ b/test/appframework/s1/appframework_suite_test.go @@ -14,6 +14,7 @@ package s1appfw import ( + "context" "os" "path/filepath" "testing" @@ -22,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/joho/godotenv" "github.com/splunk/splunk-operator/test/testenv" ) @@ -39,50 +41,72 @@ var ( testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) appListV1 []string appListV2 []string - testDataS3Bucket = os.Getenv("TEST_BUCKET") - testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation + testDataBucket = os.Getenv("TEST_BUCKET") + testBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") + appDirV1 = testenv.AppLocationV1 + appDirV2 = testenv.AppLocationV2 + pvTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) downloadDirPVTestApps = filepath.Join(currDir, "s1appfwPVTestApps-"+testenv.RandomDNSName(4)) + backend testenv.CloudStorageBackend ) // TestBasic is the main entry point func TestBasic(t *testing.T) { + if err := loadEnvFile(); err != nil { + panic("Error loading .env file: " + err.Error()) + } RegisterFailHandler(Fail) - RunSpecs(t, "Running "+testSuiteName) + sc, _ := GinkgoConfiguration() + sc.Timeout = 240 * time.Minute + + RunSpecs(t, "Running "+testSuiteName, sc) +} + +// loadEnvFile traverses up the directory tree to find a .env file +func loadEnvFile() error { + dir, err := os.Getwd() + if err != nil { + return err + } + + for { + envFile := filepath.Join(dir, ".env") + if _, err := os.Stat(envFile); err == nil { + return godotenv.Load(envFile) + } + + parentDir := filepath.Dir(dir) + if parentDir == dir { + return nil + } + dir = parentDir + } } var _ = BeforeSuite(func() { + ctx := context.TODO() var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) Expect(err).ToNot(HaveOccurred()) - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) + backend = testenv.NewCloudStorageBackend(testBucket, testDataBucket) - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") + appListV1 = testenv.BasicApps + appFileList := testenv.GetAppFileList(appListV1) - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) + appFileList = testenv.GetAppFileList(appListV2) + err = backend.DownloadFiles(ctx, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") }) var _ = AfterSuite(func() { diff --git a/test/appframework_aws/s1/appframework_aws_test.go b/test/appframework/s1/appframework_test.go similarity index 93% rename from test/appframework_aws/s1/appframework_aws_test.go rename to test/appframework/s1/appframework_test.go index c42746b91..b725f8805 100644 --- a/test/appframework_aws/s1/appframework_aws_test.go +++ b/test/appframework/s1/appframework_test.go @@ -36,7 +36,7 @@ var _ = Describe("s1appfw test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var s3TestDir string + var testDir string var uploadedApps []string var appSourceName string var appSourceVolumeName string @@ -51,7 +51,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to create testcaseenv") deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") - s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) + testDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) @@ -65,7 +65,7 @@ var _ = Describe("s1appfw test", func() { } // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) @@ -119,15 +119,15 @@ var _ = Describe("s1appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -151,7 +151,7 @@ var _ = Describe("s1appfw test", func() { // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to S3 for Standalone testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -160,7 +160,7 @@ var _ = Describe("s1appfw test", func() { // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -211,7 +211,7 @@ var _ = Describe("s1appfw test", func() { // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Standalone and Monitoring Console @@ -219,11 +219,11 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -291,23 +291,23 @@ var _ = Describe("s1appfw test", func() { appVersion := "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) appFileList := testenv.GetAppFileList(appListV2) - s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) + testDir = "s1appfw-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + testDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -330,7 +330,7 @@ var _ = Describe("s1appfw test", func() { // Create App framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -370,7 +370,7 @@ var _ = Describe("s1appfw test", func() { // ############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -381,11 +381,11 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) appFileList = testenv.GetAppFileList(appListV1) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -467,18 +467,18 @@ var _ = Describe("s1appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) - s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -500,14 +500,14 @@ var _ = Describe("s1appfw test", func() { // testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) // Upload apps to S3 for Standalone - s3TestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + testDir := "s1appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -653,19 +653,19 @@ var _ = Describe("s1appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Standalone - s3TestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + testDir := "s1appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -717,7 +717,7 @@ var _ = Describe("s1appfw test", func() { // ############## UPGRADE APPS ################# // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Standalone and Monitoring Console @@ -725,7 +725,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -767,18 +767,18 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to S3 testcaseEnvInst.Log.Info("Upload ES app on S3") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, testDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ Type: enterpriseApi.PremiumAppsTypeEs, EsDefaults: enterpriseApi.EsDefaults{ @@ -818,18 +818,18 @@ var _ = Describe("s1appfw test", func() { // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Download ES App from S3 testcaseEnvInst.Log.Info("Download updated ES app from S3") - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + err = backend.DownloadFiles(ctx, appDirV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to S3 for Standalone appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Es app to S3 for Standalone and Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Es app to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -878,19 +878,19 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 testcaseEnvInst.Log.Info("Upload bigger amount of apps to S3 for this test") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -963,15 +963,15 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 0) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -993,13 +993,13 @@ var _ = Describe("s1appfw test", func() { testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) // Upload V1 apps to S3 - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 0) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 0) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1037,18 +1037,18 @@ var _ = Describe("s1appfw test", func() { //############### UPGRADE APPS ################ //Delete apps on S3 for new Apps - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil //Upload new Versioned Apps to S3 appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1141,27 +1141,27 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download the extra apps from S3 for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 for first Standalone testcaseEnvInst.Log.Info("Upload apps to S3 for 1st Standalone") appFileListStandalone1 := testenv.GetAppFileList(appList1) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileListStandalone1, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileListStandalone1, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for second Standalone testcaseEnvInst.Log.Info("Upload apps to S3 for 2nd Standalone") - s3TestDirStandalone2 := "s1appfw-2-" + testenv.RandomDNSName(4) + testDirStandalone2 := "s1appfw-2-" + testenv.RandomDNSName(4) appFileListStandalone2 := testenv.GetAppFileList(appList2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirStandalone2, appFileListStandalone2, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDirStandalone2, appFileListStandalone2, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1176,7 +1176,7 @@ var _ = Describe("s1appfw test", func() { // Create App framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, s3TestDirStandalone2, 60) + appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, testDirStandalone2, 60) specStandalone2 := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1238,15 +1238,15 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1271,20 +1271,20 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1311,7 +1311,7 @@ var _ = Describe("s1appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Standalone") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1350,18 +1350,18 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1447,18 +1447,18 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1522,7 +1522,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1531,7 +1531,7 @@ var _ = Describe("s1appfw test", func() { // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1567,7 +1567,7 @@ var _ = Describe("s1appfw test", func() { testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) // Disable the app - testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDir) + testenv.DisableAppsToS3(downloadDirV1, appFileName, testDir) // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) @@ -1580,8 +1580,8 @@ var _ = Describe("s1appfw test", func() { testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, []string{standalonePodName}, testcaseEnvInst.GetName(), appName, "disabled", false) // Delete the file from S3 - s3Filepath := filepath.Join(s3TestDir, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + s3Filepath := filepath.Join(testDir, appFileName[0]) + err = testenv.DeleteFileOnS3(testBucket, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -1632,7 +1632,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1641,7 +1641,7 @@ var _ = Describe("s1appfw test", func() { // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1686,7 +1686,7 @@ var _ = Describe("s1appfw test", func() { // ############## UPGRADE APPS ################# // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + backend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Standalone @@ -1694,7 +1694,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1751,18 +1751,18 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to S3 for Standalone testcaseEnvInst.Log.Info("Upload apps to S3 for Standalone") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 120) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 120) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1787,7 +1787,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s app to S3 for Standalone", appVersion)) appFileList = testenv.GetAppFileList([]string{appListV2[0]}) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s app to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1843,12 +1843,12 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = backend.DownloadFiles(ctx, pvTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirPVTestApps) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1857,7 +1857,7 @@ var _ = Describe("s1appfw test", func() { // Create App framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -1910,18 +1910,18 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := backend.DownloadFiles(ctx, appDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to S3 for Standalone testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1985,15 +1985,15 @@ var _ = Describe("s1appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + testDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) + uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) + appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -2021,7 +2021,7 @@ var _ = Describe("s1appfw test", func() { // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to S3 for Standalone testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = backend.UploadFiles(ctx, testDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2030,7 +2030,7 @@ var _ = Describe("s1appfw test", func() { // Create App framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) + appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, testDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ diff --git a/test/appframework_az/c3/appframework_azure_suite_test.go b/test/appframework_az/c3/appframework_azure_suite_test.go deleted file mode 100644 index 313c7c4fc..000000000 --- a/test/appframework_az/c3/appframework_azure_suite_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azurec3appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go deleted file mode 100644 index f6fc5bf5b..000000000 --- a/test/appframework_az/c3/appframework_azure_test.go +++ /dev/null @@ -1,2857 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurec3appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDirShc string - var azTestDirIdxc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - var backend testenv.CloudStorageBackend - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - backend = testenv.NewCloudStorageBackend("", "") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to cloud storage - if !testcaseEnvInst.SkipTeardown { - backend.DeleteFiles(ctx, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - testenv.C3AppFrameworkUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - testenv.C3AppFrameworkDowngradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - time.Sleep(60 * time.Second) - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to S3 - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on S3 location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Azure - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Download Technology add-on app from S3 - testcaseEnvInst.Log.Info("Download Technology add-on app from Axure") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Create directory for file upload to S3 - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app to Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to S3 for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileListIdxc) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - - // Deploy C3 SVA - // Deploy the Cluster Master - testcaseEnvInst.Log.Info("Deploy Cluster Master") - cmSpec := enterpriseApiV3.ClusterMasterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterMasterWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterMasterRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shc.Spec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### UPGRADE APPS ################ - // Delete ES app on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Download ES App from Azure - appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 ES app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = esApp - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - time.Sleep(2 * time.Minute) // FIXME adding sleep to see if verification succeedes - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Azure (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Azure for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Azure", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - //Delete the file from az - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with clusterMasterRef - ############# VERIFICATION ################# - * Verify clusterMasterRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with clusterMasterRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterMasterRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with clusterMasterRef - testcaseEnvInst.Log.Info("Deploy Standalone with clusterMasterRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Ensure that the Standalone goes to Ready phase - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) -}) diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go deleted file mode 100644 index 4412efe43..000000000 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ /dev/null @@ -1,3268 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurec3appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDirShc string - var azTestDirIdxc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - //uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - time.Sleep(60 * time.Second) - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to S3 - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on S3 location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Azure - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Download Technology add-on app from S3 - testcaseEnvInst.Log.Info("Download Technology add-on app from Axure") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Create directory for file upload to S3 - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app to Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to S3 for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileListIdxc) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - - // Deploy C3 SVA - // Deploy the Cluster Master - testcaseEnvInst.Log.Info("Deploy Cluster Master") - cmSpec := enterpriseApi.ClusterManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shc.Spec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### UPGRADE APPS ################ - // Delete ES app on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Download ES App from Azure - appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 ES app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Es app to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = esApp - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - time.Sleep(2 * time.Minute) // FIXME adding sleep to see if verification succeedes - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Azure (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Azure for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Azure", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - //Delete the file from az - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with ClusterManagerRef - ############# VERIFICATION ################# - * Verify ClusterManagerRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with ClusterManagerRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with ClusterManagerRef - testcaseEnvInst.Log.Info("Deploy Standalone with ClusterManagerRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with ClusterManagerRef") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Ensure that the Standalone goes to Ready phase - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) -}) diff --git a/test/appframework_az/m4/appframework_azure_suite_test.go b/test/appframework_az/m4/appframework_azure_suite_test.go deleted file mode 100644 index 6184e543b..000000000 --- a/test/appframework_az/m4/appframework_azure_suite_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azurem4appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/m4/appframework_azure_test.go b/test/appframework_az/m4/appframework_azure_test.go deleted file mode 100644 index 1a9af4953..000000000 --- a/test/appframework_az/m4/appframework_azure_test.go +++ /dev/null @@ -1,2691 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurem4appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var azTestDirShc string - var azTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify MC is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - time.Sleep(2 * time.Minute) - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster master with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - time.Sleep(60 * time.Second) - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait 5 seconds to be sure reconcile caused by CR update and config map update are done - testcaseEnvInst.Log.Info("Wait 5 seconds to be sure reconcile caused by CR update and config map update are done") - time.Sleep(5 * time.Second) - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Wait for the end of AppsRepoPollInterval duration - testcaseEnvInst.Log.Info("Wait for the end of AppsRepoPollInterval duration") - time.Sleep(2 * time.Minute) - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - -}) diff --git a/test/appframework_az/m4/manager_appframework_azure_test.go b/test/appframework_az/m4/manager_appframework_azure_test.go deleted file mode 100644 index 9cd5d9507..000000000 --- a/test/appframework_az/m4/manager_appframework_azure_test.go +++ /dev/null @@ -1,2690 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurem4appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var azTestDirShc string - var azTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify MC is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - time.Sleep(2 * time.Minute) - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("m4azure, integration, managerappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - time.Sleep(60 * time.Second) - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait 5 seconds to be sure reconcile caused by CR update and config map update are done - testcaseEnvInst.Log.Info("Wait 5 seconds to be sure reconcile caused by CR update and config map update are done") - time.Sleep(5 * time.Second) - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Wait for the end of AppsRepoPollInterval duration - testcaseEnvInst.Log.Info("Wait for the end of AppsRepoPollInterval duration") - time.Sleep(2 * time.Minute) - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("m4azure, integration, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - -}) diff --git a/test/appframework_az/s1/appframework_azure_suite_test.go b/test/appframework_az/s1/appframework_azure_suite_test.go deleted file mode 100644 index d3c06c3c4..000000000 --- a/test/appframework_az/s1/appframework_azure_suite_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azures1appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/s1/appframework_azure_test.go b/test/appframework_az/s1/appframework_azure_test.go deleted file mode 100644 index 933b70fa8..000000000 --- a/test/appframework_az/s1/appframework_azure_test.go +++ /dev/null @@ -1,2016 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azures1appfw - -import ( - "context" - "fmt" - "path/filepath" - "strings" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("s1appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDir string - var uploadedApps []string - var appSourceName string - var appSourceVolumeName string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - azTestDir = "s1appfw-" + testenv.RandomDNSName(4) - appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframeworkazure, azure_sanity: can deploy a Standalone instance with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - - azTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - appFrameworkSpecMC.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - azTestDir = "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - - uploadedApps = nil - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smoke, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ INITIAL VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############# DOWNGRADE APPS ################ - * Upload V1 apps on Azure - * Wait for Monitoring Console and Standalone pods to be ready - ########## DOWNGRADE VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V2 apps to Azure - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - azTestDir = "azures1appfw-" + testenv.RandomDNSName(4) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to Azure for Standalone and Monitoring Console - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV1) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV1 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework, azure_sanity: can deploy a Standalone instance with App Framework enabled, install apps, scale up, install apps on new pod, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on Azure - * Create 2 app sources for Monitoring Console and Standalone - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING UP VERIFICATION ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############## SCALING DOWN ################# - * Scale down Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING DOWN VERIFICATION ####### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Standalone and Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload apps to Azure for Standalone - azTestDir := "azures1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - //########### SCALING UP VERIFICATION ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############## SCALING DOWN ################# - // Scale down Standalone instance - testcaseEnvInst.Log.Info("Scale down Standalone") - scaledReplicaCount = 1 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale down Standalone") - - // Ensure Standalone is scaling down - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingDown) - - // Wait for Standalone to be in READY status - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - //########### SCALING DOWN VERIFICATION ####### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps, scale up, upgrade apps", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on Azure - * Create app source for Standalone - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Standalone to be ready - ############### UPGRADE APPS ################ - * Upload V2 apps to Azure App Source - ###### SCALING UP/UPGRADE VERIFICATIONS ##### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Standalone - azTestDir := "azures1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // ############## UPGRADE APPS ################# - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ SCALING UP/UPGRADE VERIFICATIONS ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - // ES App Installation not supported at the time. Will be added back at a later time. - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework: can deploy a Standalone and have ES app installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to Azure - * Create App Source for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Standalone - */ - - //################## SETUP #################### - - // Download ES App from Azure - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app on Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) - appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - appVersion := "V1" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - - // Download ES App from Azure - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 apps to S3 for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = esApp - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled and install around 350MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create app source for Standalone - * Add more apps than usual on Azure for this test - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a bigger list of apps to be installed than the default one - appList := append(appListV1, testenv.RestartNeededApps...) - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to Azure - testcaseEnvInst.Log.Info("Upload bigger amount of apps to Azure for this test") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, smoke, appframeworkazures1, appframework: can deploy a standalone instance with App Framework enabled for manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Create app source for Standalone - * Prepare and deploy Standalone with app framework(MANUAL POLL) and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create Standalone Deployment with App Framework - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### UPGRADE APPS ################ - - //Delete apps on Azure for new Apps - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - //Upload new Versioned Apps to Azure - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Config map data for", "Standalone", config.Data["Standalone"]) - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["Standalone"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - config.Data["MonitoringConsole"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["Standalone"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - //############### VERIFICATION FOR UPGRADE ################ - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy Several standalone CRs in the same namespace with App Framework enabled", func() { - - /* Test Steps - ################## SETUP #################### - * Add more apps than usual on Azure for this test - * Split the App list into 2 segments with a common apps and uncommon apps for each Standalone - * Create app source for 2 Standalones - * Prepare and deploy Standalones with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a list of apps to be installed on both standalone - appList1 := append(appListV1, testenv.RestartNeededApps[len(testenv.RestartNeededApps)/2:]...) - appList2 := append(appListV1, testenv.RestartNeededApps[:len(testenv.RestartNeededApps)/2]...) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download the extra apps from Azure for this test") - appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to Azure for first Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for 1st Standalone") - appFileListStandalone1 := testenv.GetAppFileList(appList1) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileListStandalone1) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for second Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for 2nd Standalone") - azTestDirStandalone2 := "azures1appfw-2-" + testenv.RandomDNSName(4) - appFileListStandalone2 := testenv.GetAppFileList(appList2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirStandalone2, appFileListStandalone2) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create App framework Spec - appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) - specStandalone2 := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecStandalone2, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy 1st Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy 1st Standalone instance") - testcaseEnvInst.Log.Info("Deploy 2nd Standalone") - standalone2Name := deployment.GetName() + testenv.RandomDNSName(3) - standalone2, err := deployment.DeployStandaloneWithGivenSpec(ctx, standalone2Name, specStandalone2) - Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone2, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList1, CrAppFileList: appFileListStandalone1} - standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} - standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can add new apps to app source while install is in progress and have all apps installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Standalone - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Standalone") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Verify all apps are installed on Standalone - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app install is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App download is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ Upload Disabled App ########### - * Download disabled app from Azure - * Delete the app from Azure - * Check for repo state in App Deployment Info - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDir) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Wait for App state to update after config file change - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, []string{standalonePodName}, testcaseEnvInst.GetName(), appName, "disabled", false) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDir, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, attempt to update using incorrect Azure credentials", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec - * Check for changes in App phase to determine if next poll has been triggered - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - * Check no apps are updated as auth key is incorrect - ############ Modify secret key to correct one########### - * Apply spec with correct credentails - * Wait for the pod to be ready - ############ V2 APP VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - secretStruct, _ := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) - secretData := secretStruct.Data - modifiedSecretData := map[string][]byte{"azure_sa_name": []byte(testenv.RandomDNSName(5)), "azure_sa_secret_key": []byte(testenv.RandomDNSName(5))} - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec - testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // ############## UPGRADE APPS ################# - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Check no apps are updated as auth key is incorrect - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Modify secret key to correct one########### - // Apply spec with correct credentials - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, secretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is completed, upload new versions of the apps - ############## VERIFICATIONS ################ - * Verify App download is in completed on Standalone - * Upload updated app to Azure as pervious app download is complete - * Verify app is installed on Standalone - ############## UPGRADE VERIFICATIONS ################ - * Wait for next poll to trigger on Standalone - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download test app from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload apps to Azure for Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s app to Azure for Standalone", appVersion)) - appFileList = testenv.GetAppFileList([]string{appListV2[0]}) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s app to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - //############ UPGRADE VERIFICATION ########### - appVersion = "V2" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload file to Azure for standalone - * Create app source for Standalone with parallelDownload=15 - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 15 - - // Create App framework Spec - appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * When app download is complete, delete apps from app directory - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download big size apps from Azure - appVersion := "V1" - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big app") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App Download is completed on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smoke, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps and check isDeploymentInProgress is set for Standaloen and MC CR's", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, mcName, mc.Kind) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - }) - }) -}) diff --git a/test/appframework_gcp/c3/appframework_gcs_suite_test.go b/test/appframework_gcp/c3/appframework_gcs_suite_test.go deleted file mode 100644 index 9aa061bad..000000000 --- a/test/appframework_gcp/c3/appframework_gcs_suite_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package c3gcpappfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "c3appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to Gcs - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Gcs after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files.") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files.") -}) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go deleted file mode 100644 index 503b54b10..000000000 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package c3gcpappfw - -import ( - "context" - "fmt" - "path/filepath" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - testenv "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - - var deployment *testenv.Deployment - var gcsTestDirShc string - var gcsTestDirIdxc string - //var gcsTestDirShcLocal string - //var gcsTestDirIdxcLocal string - //var gcsTestDirShcCluster string - //var gcsTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - var backend testenv.CloudStorageBackend - - ctx := context.TODO() - - BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 5000 - backend = testenv.NewCloudStorageBackend(testGcsBucket, testDataGcsBucket) - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to cloud storage - if !testcaseEnvInst.SkipTeardown { - backend.DeleteFiles(ctx, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - testenv.C3AppFrameworkUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - testenv.C3AppFrameworkDowngradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - testenv.C3AppFrameworkLocalScopeUpgradeTest(ctx, backend, testcaseEnvInst, deployment, appListV1, appListV2, downloadDirV1, downloadDirV2, &uploadedApps) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) -}) diff --git a/test/appframework_gcp/c3/manager_appframework_test.go b/test/appframework_gcp/c3/manager_appframework_test.go deleted file mode 100644 index 66c553e47..000000000 --- a/test/appframework_gcp/c3/manager_appframework_test.go +++ /dev/null @@ -1,3420 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package c3gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - - var deployment *testenv.Deployment - var gcsTestDirShc string - var gcsTestDirIdxc string - var gcsTestDirShcLocal string - var gcsTestDirIdxcLocal string - var gcsTestDirShcCluster string - var gcsTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - testenv.SpecifiedTestTimeout = 100000 - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Gcs - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to Gcs for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework and Image Upgrade", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade the image and apps", func() { - - //################## SETUP #################### - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from Gcs") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Upload V1 apps to Gcs for Monitoring Console - oldImage := "Refer to RELATED_SPLUNK_IMAGE_ENTERPRISE" - newImage := "splunk/splunk:latest" - - lm, err := deployment.DeployLicenseManager(ctx, deployment.GetName()) - cm, err := deployment.DeployClusterManager(ctx, deployment.GetName(), lm.GetName(), "", "") - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: cm.GetName(), - }, - }, - } - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - shcName := fmt.Sprintf("%s-shc", deployment.GetName()) - idxName := fmt.Sprintf("%s-idxc", deployment.GetName()) - shc, err := deployment.DeploySearchHeadCluster(ctx, shcName, cm.GetName(), lm.GetName(), "", mcName) - idxc, err := deployment.DeployIndexerCluster(ctx, idxName, lm.GetName(), 3, cm.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") - - // Wait for License Manager to be in READY phase - testenv.LicenseManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Monitoring Console goes to Ready phase - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE IMAGE ################ - - // Update LM Image - - testcaseEnvInst.Log.Info("Upgrading the License Manager Image", "Current Image", oldImage, "New Image", newImage) - lm.Spec.Image = newImage - err = deployment.UpdateCR(ctx, lm) - Expect(err).To(Succeed(), "Failed upgrade License Manager image") - - // Update CM image - - testcaseEnvInst.Log.Info("Upgrading the Cluster Manager Image", "Current Image", oldImage, "New Image", newImage) - cm.Spec.Image = newImage - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed upgrade Cluster Manager image") - - // Update MC image - - testcaseEnvInst.Log.Info("Upgrading the Monitoring Console Image", "Current Image", oldImage, "New Image", newImage) - mc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, mc) - Expect(err).To(Succeed(), "Failed upgrade Monitoring Console image") - - // Update SHC image - - testcaseEnvInst.Log.Info("Upgrading the Search Head Cluster Image", "Current Image", oldImage, "New Image", newImage) - shc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed upgrade Search Head Cluster image") - - // // Update IDXC image - - testcaseEnvInst.Log.Info("Upgrading the Indexer Cluster Image", "Current Image", oldImage, "New Image", newImage) - idxc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed upgrade Indexer Cluster image") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for License Manager to be in READY phase - testenv.LicenseManagerReady(ctx, deployment, testcaseEnvInst) - - // // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Gcs - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Gcs for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - time.Sleep(60 * time.Second) - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - time.Sleep(2 * time.Minute) // FIXME adding sleep to see if verification succeedes - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Gcs (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Gcs for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from Gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Gcs for Indexer Cluster - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Gcs for Indexer Cluster - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Gcs", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Gcs for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Gcs for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - // Vivek need testing - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Gcs for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Gcs for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - // Vivek need testing - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from Gcs - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Gcs bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Gcs - gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Gcs test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Gcs bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Gcs for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to Gcs for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search head Cluster", appVersion)) - gcsTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Gcs - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Indexer Cluster") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with clusterMasterRef - ############# VERIFICATION ################# - * Verify clusterMasterRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with clusterMasterRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with clusterMasterRef - testcaseEnvInst.Log.Info("Deploy Standalone with clusterManagerRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Ensure that the Standalone goes to Ready phase - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to Gcs - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on Gcs location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Gcs - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Gcs") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") - - // Download Technology add-on app from Gcs - testcaseEnvInst.Log.Info("Download Technology add-on app from Gcs") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") - - // Create directory for file upload to Gcs - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Gcs - testcaseEnvInst.Log.Info("Upload ES app to Gcs") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to Gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileListIdxc, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) - - // Deploy C3 SVA - // Deploy the Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - cmSpec := enterpriseApi.ClusterManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shSpec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // //############### UPGRADE APPS ################ - // // Download ES App from Gcs - // appVersion = "V2" - // testcaseEnvInst.Log.Info("Download updated ES app from Gcs") - // err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - // Expect(err).To(Succeed(), "Unable to download ES app") - - // // Upload V2 ES app to Gcs for Search Head Cluster - // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Gcs for Search Head Cluster", appVersion)) - // uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Gcs test directory for Search Head Cluster", appVersion)) - // uploadedApps = append(uploadedApps, uploadedFiles...) - - // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // // Ensure that the Cluster Manager goes to Ready phase - // testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // // Ensure Indexers go to Ready phase - // testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Ensure Search Head Cluster go to Ready phase - // testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // // Verify RF SF is met - // testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // // Get Pod age to check for pod resets later - // splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // //############ FINAL VERIFICATIONS ############ - - // shcAppSourceInfo.CrAppVersion = appVersion - // shcAppSourceInfo.CrAppList = esApp - // shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - // allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - // testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) -}) diff --git a/test/appframework_gcp/m4/appframework_gcs_suite_test.go b/test/appframework_gcp/m4/appframework_gcs_suite_test.go deleted file mode 100644 index 8f4a28249..000000000 --- a/test/appframework_gcp/m4/appframework_gcs_suite_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package m4gcpappfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "m4appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - testenvInstance.Log.Info("logging download details", "bucket", testDataGcsBucket, "gcsAppDirV1", gcsAppDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_gcp/m4/appframework_gcs_test.go b/test/appframework_gcp/m4/appframework_gcs_test.go deleted file mode 100644 index e7276d4f6..000000000 --- a/test/appframework_gcp/m4/appframework_gcs_test.go +++ /dev/null @@ -1,2709 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package m4gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var gcsTestDirShc string - var gcsTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var gcsTestDirShcLocal string - var gcsTestDirIdxcLocal string - var gcsTestDirShcCluster string - var gcsTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - _ = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify MC is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to GCP for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to GCP for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for M4 - * Create app source for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to GCP - * Create app source with manaul poll for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Master and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - time.Sleep(2 * time.Minute) - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Master and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCP - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to GCP (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to GCP (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster master with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to GCP (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to GCP (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Master and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Master - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Master - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Master - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - time.Sleep(60 * time.Second) - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Master and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Master - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Master - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from gcs - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from GCP - gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCP test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from GCP bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait 5 seconds to be sure reconcile caused by CR update and config map update are done - testcaseEnvInst.Log.Info("Wait 5 seconds to be sure reconcile caused by CR update and config map update are done") - time.Sleep(5 * time.Second) - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Wait for the end of AppsRepoPollInterval duration - testcaseEnvInst.Log.Info("Wait for the end of AppsRepoPollInterval duration") - time.Sleep(2 * time.Minute) - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to GCP for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - gcsTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) - gcsTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from GCP - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to GCP for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to GCP for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) -}) diff --git a/test/appframework_gcp/m4/manager_appframework_test.go b/test/appframework_gcp/m4/manager_appframework_test.go deleted file mode 100644 index 24730de60..000000000 --- a/test/appframework_gcp/m4/manager_appframework_test.go +++ /dev/null @@ -1,2708 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package m4gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var s3TestDirShc string - var s3TestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var s3TestDirShcLocal string - var s3TestDirIdxcLocal string - var s3TestDirShcCluster string - var s3TestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - _ = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify MC is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to GCP for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to GCP for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to GCP - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - time.Sleep(2 * time.Minute) - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCP - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to GCP (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to GCP (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - s3TestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to GCP (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to GCP (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - time.Sleep(60 * time.Second) - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from s3 - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testenv.DisableAppsToGCP(downloadDirV1, appFileName, s3TestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from GCP - s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, s3Filepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCP test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from GCP bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Manager and Search Head Cluster - * Change status to 'ON' in config map for Cluster Manager and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Manager CR with latest config - cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") - - // Set AppsRepoPollInterval for Cluster Manager to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Manager") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Manager") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Manager") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait 5 seconds to be sure reconcile caused by CR update and config map update are done - testcaseEnvInst.Log.Info("Wait 5 seconds to be sure reconcile caused by CR update and config map update are done") - time.Sleep(5 * time.Second) - - // Verify status is 'ON' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Manager and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Wait for the end of AppsRepoPollInterval duration - testcaseEnvInst.Log.Info("Wait for the end of AppsRepoPollInterval duration") - time.Sleep(2 * time.Minute) - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to GCP for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - s3TestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) - s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from GCP - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to GCP for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to GCP for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) -}) diff --git a/test/appframework_gcp/s1/appframework_gcs_suite_test.go b/test/appframework_gcp/s1/appframework_gcs_suite_test.go deleted file mode 100644 index af2fab4c2..000000000 --- a/test/appframework_gcp/s1/appframework_gcs_suite_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package gcps1appfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGCSBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "s1appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_gcp/s1/appframework_gcs_test.go b/test/appframework_gcp/s1/appframework_gcs_test.go deleted file mode 100644 index 2f23abb3f..000000000 --- a/test/appframework_gcp/s1/appframework_gcs_test.go +++ /dev/null @@ -1,2055 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package gcps1appfw - -import ( - "context" - "fmt" - "path/filepath" - "strings" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("s1appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var gcsTestDir string - var uploadedApps []string - var appSourceName string - var appSourceVolumeName string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) - appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to gcs for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to gcs App Source - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to gcs for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to gcs for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V2 apps to gcs for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ INITIAL VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############# DOWNGRADE APPS ################ - * Upload V1 apps on gcs - * Wait for Monitoring Console and Standalone pods to be ready - ########## DOWNGRADE VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V2 apps to gcs - appVersion := "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList := testenv.GetAppFileList(appListV2) - gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############# DOWNGRADE APPS ################ - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to gcs for Standalone and Monitoring Console - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV1) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV1 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, smokegcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install apps, scale up, install apps on new pod, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on gcs - * Create 2 app sources for Monitoring Console and Standalone - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING UP VERIFICATION ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############## SCALING DOWN ################# - * Scale down Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING DOWN VERIFICATION ####### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to gcs for Standalone and Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - // testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload apps to gcs for Standalone - gcsTestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - //########### SCALING UP VERIFICATION ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //############## SCALING DOWN ################# - // Scale down Standalone instance - testcaseEnvInst.Log.Info("Scale down Standalone") - scaledReplicaCount = 1 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale down Standalone") - - // Ensure Standalone is scaling down - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingDown) - - // Wait for Standalone to be in READY status - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - //########### SCALING DOWN VERIFICATION ####### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, integrationgcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install apps, scale up, upgrade apps", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on gcs - * Create app source for Standalone - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Standalone to be ready - ############### UPGRADE APPS ################ - * Upload V2 apps to gcs App Source - ###### SCALING UP/UPGRADE VERIFICATIONS ##### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to gcs for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to gcs for Standalone - gcsTestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // ############## UPGRADE APPS ################# - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to gcs for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ SCALING UP/UPGRADE VERIFICATIONS ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - // ES App Installation not supported at the time. Will be added back at a later time. - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, integrationgcp, appframeworkgcp: can deploy a Standalone and have ES app installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to gcs - * Create App Source for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Standalone - */ - - //################## SETUP #################### - - // Download ES App from gcs - testcaseEnvInst.Log.Info("Download ES app from gcs") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload ES app to gcs - testcaseEnvInst.Log.Info("Upload ES app on gcs") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) - appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") - - // Ensure Standalone goes to Ready phase - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - appVersion := "V1" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Download ES App from gcs - testcaseEnvInst.Log.Info("Download updated ES app from gcs") - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 apps to gcs for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Es app to gcs for Standalone and Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Es app to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = esApp - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled and install around 350MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create app source for Standalone - * Add more apps than usual on gcs for this test - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a bigger list of apps to be installed than the default one - appList := append(appListV1, testenv.RestartNeededApps...) - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to GCS - testcaseEnvInst.Log.Info("Upload bigger amount of apps to GCS for this test") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, smokegcp, appframeworkgcp: can deploy a standalone instance with App Framework enabled for manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Create app source for Standalone - * Prepare and deploy Standalone with app framework(MANUAL POLL) and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to GCS App Source - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCS - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create Standalone Deployment with App Framework - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### UPGRADE APPS ################ - - //Delete apps on GCS for new Apps - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - //Upload new Versioned Apps to GCS - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Config map data for", "Standalone", config.Data["Standalone"]) - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["Standalone"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - config.Data["MonitoringConsole"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["Standalone"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - //############### VERIFICATION FOR UPGRADE ################ - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy Several standalone CRs in the same namespace with App Framework enabled", func() { - - /* Test Steps - ################## SETUP #################### - * Add more apps than usual on GCS for this test - * Split the App list into 2 segments with a common apps and uncommon apps for each Standalone - * Create app source for 2 Standalones - * Prepare and deploy Standalones with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a list of apps to be installed on both standalone - appList1 := append(appListV1, testenv.RestartNeededApps[len(testenv.RestartNeededApps)/2:]...) - appList2 := append(appListV1, testenv.RestartNeededApps[:len(testenv.RestartNeededApps)/2]...) - appVersion := "V1" - - // Download apps from GCS - testcaseEnvInst.Log.Info("Download the extra apps from GCS for this test") - appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to GCS for first Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for 1st Standalone") - appFileListStandalone1 := testenv.GetAppFileList(appList1) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileListStandalone1, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCS for second Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for 2nd Standalone") - gcsTestDirStandalone2 := "s1appfw-2-" + testenv.RandomDNSName(4) - appFileListStandalone2 := testenv.GetAppFileList(appList2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirStandalone2, appFileListStandalone2, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create App framework Spec - appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) - specStandalone2 := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecStandalone2, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy 1st Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy 1st Standalone instance") - testcaseEnvInst.Log.Info("Deploy 2nd Standalone") - standalone2Name := deployment.GetName() + testenv.RandomDNSName(3) - standalone2, err := deployment.DeployStandaloneWithGivenSpec(ctx, standalone2Name, specStandalone2) - Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone2, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList1, CrAppFileList: appFileListStandalone1} - standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} - standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can add new apps to app source while install is in progress and have all apps installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCS for Standalone - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCS for Standalone") - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Verify all apps are installed on Standalone - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app install is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - if err != nil { - for i := 1; i < 10; i++ { - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - if err == nil { - continue - } else { - time.Sleep(1 * time.Second) - } - } - } - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - // Verify App installation is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App download is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install an app then disable it and remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - * Disable the app - * Delete the app from GCS - * Check for repo state in App Deployment Info - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) - - // Disable the app - testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDir) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Wait for App state to update after config file change - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, []string{standalonePodName}, testcaseEnvInst.GetName(), appName, "disabled", false) - - // Delete the file from GCS - gcsFilepath := filepath.Join(gcsTestDir, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGCSBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCS test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from GCS bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, attempt to update using incorrect GCS credentials", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec - * Check for changes in App phase to determine if next poll has been triggered - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to GCS App Source - * Check no apps are updated as auth key is incorrect - ############ Modify secret key to correct one########### - * Apply spec with correct credentails - * Wait for the pod to be ready - ############ V2 APP VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) - Expect(err).To(Succeed(), "Unable to obtain secret object") - secretData := secretStruct.Data - modifiedSecretData := map[string][]byte{"gcs_access_key": []byte(testenv.RandomDNSName(5)), "gcs_secret_key": []byte(testenv.RandomDNSName(5))} - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec - testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // ############## UPGRADE APPS ################# - // Delete apps on - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCS for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Check no apps are updated as auth key is incorrect - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Modify secret key to correct one########### - // Apply spec with correct credentials - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, secretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is completed, upload new versions of the apps - ############## VERIFICATIONS ################ - * Verify App download is in completed on Standalone - * Upload updated app to GCS as pervious app download is complete - * Verify app is installed on Standalone - ############## UPGRADE VERIFICATIONS ################ - * Wait for next poll to trigger on Standalone - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download test app from GCS - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload apps to GCS for Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCS for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s app to GCS for Standalone", appVersion)) - appFileList = testenv.GetAppFileList([]string{appListV2[0]}) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s app to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - appVersion = "V2" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy a Standalone instance and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload file to gcs for standalone - * Create app source for Standalone with parallelDownload=15 - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 15 - - // Create App framework Spec - appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * When app download is complete, delete apps from app directory - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download big size apps from GCS - appVersion := "V1" - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big app") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App Download is completed on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps and check isDeploymentInProgress is set for Standaloen and MC CR's", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, mcName, mc.Kind) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - }) - }) - -}) diff --git a/test/deploy-cluster.sh b/test/deploy-cluster.sh index e8751086f..ecfe99e3b 100755 --- a/test/deploy-cluster.sh +++ b/test/deploy-cluster.sh @@ -6,10 +6,6 @@ source ${scriptdir}/env.sh action=${1:-up} -if [[ -z "${TEST_CLUSTER_PLATFORM}" ]]; then - echo "Test Cluster Platform Not Set in Environment Variables. Changing to env.sh value" - export TEST_CLUSTER_PLATFORM="${CLUSTER_PROVIDER}" -fi if [[ -z "${TEST_CLUSTER_NAME}" ]]; then echo "Test Cluster Name Not Set in Environment Variables. Changing to env.sh value" @@ -22,10 +18,10 @@ if [[ -z "${CLUSTER_WORKERS}" ]]; then fi # source the script file containing the create/delete cluster implementation -srcfile="${scriptdir}/deploy-${TEST_CLUSTER_PLATFORM}-cluster.sh" +srcfile="${scriptdir}/deploy-${CLUSTER_PROVIDER}-cluster.sh" if [ ! -f "$srcfile" ]; then - echo "Unknown cluster provider '${TEST_CLUSTER_PLATFORM}'" + echo "Unknown cluster provider '${CLUSTER_PROVIDER}'" exit 1 fi source $srcfile diff --git a/test/env.sh b/test/env.sh index 942b91e85..b624d2cc9 100644 --- a/test/env.sh +++ b/test/env.sh @@ -57,7 +57,6 @@ : "${DEBUG_RUN:=False}" # Type of deplyoment, manifest files or helm chart, possible values "manifest" or "helm" : "${DEPLOYMENT_TYPE:=manifest}" -: "${TEST_CLUSTER_PLATFORM:=eks}" # Docker registry to use to push the test images to and pull from in the cluster if [ -z "${PRIVATE_REGISTRY}" ]; then diff --git a/test/testenv/appframework_c3_testflows.go b/test/testenv/appframework_c3_testflows.go deleted file mode 100644 index f573c7463..000000000 --- a/test/testenv/appframework_c3_testflows.go +++ /dev/null @@ -1,412 +0,0 @@ -package testenv - -import ( - "context" - "fmt" - - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - corev1 "k8s.io/api/core/v1" -) - -// C3AppFrameworkUpgradeTest deploys a C3 SVA with App Framework enabled, -// installs V1 apps (with MC), then upgrades them to V2 and re-verifies. -// This is the shared flow behind the "upgrade" smoke test in every cloud provider. -func C3AppFrameworkUpgradeTest( - ctx context.Context, - backend CloudStorageBackend, - testcaseEnvInst *TestCaseEnv, - deployment *Deployment, - appListV1, appListV2 []string, - downloadDirV1, downloadDirV2 string, - uploadedApps *[]string, -) { - appVersion := "V1" - appFileList := GetAppFileList(appListV1) - - // Upload V1 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) - testDirMC := "c3appfw-mc-" + RandomDNSName(4) - uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + RandomDNSName(3) - appFrameworkSpecMC := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) - testDirIdxc := "c3appfw-idxc-" + RandomDNSName(4) - uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - // Upload V1 apps for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) - testDirShc := "c3appfw-shc-" + RandomDNSName(4) - uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + RandomDNSName(3) - appFrameworkSpecIdxc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) - appFrameworkSpecShc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) - - resourceVersion := GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - ClusterMasterReady(ctx, deployment, testcaseEnvInst) - SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - splunkPodAge := GetPodsStartTime(testcaseEnvInst.GetName()) - - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := DumpGetPods(testcaseEnvInst.GetName()) - VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Initial verifications - idxcPodNames := GeneratePodNameSlice(IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := GeneratePodNameSlice(SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Upgrade apps - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) - backend.DeleteFiles(ctx, *uploadedApps) - *uploadedApps = nil - - resourceVersion = GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - appVersion = "V2" - appFileList = GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) - uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) - uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) - uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - ClusterMasterReady(ctx, deployment, testcaseEnvInst) - SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - splunkPodAge = GetPodsStartTime(testcaseEnvInst.GetName()) - - // Final verifications - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) - allAppSourceInfo = []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) -} - -// C3AppFrameworkDowngradeTest deploys a C3 SVA with App Framework enabled, -// installs V2 apps (with MC), then downgrades them to V1 and re-verifies. -func C3AppFrameworkDowngradeTest( - ctx context.Context, - backend CloudStorageBackend, - testcaseEnvInst *TestCaseEnv, - deployment *Deployment, - appListV1, appListV2 []string, - downloadDirV1, downloadDirV2 string, - uploadedApps *[]string, -) { - appVersion := "V2" - appFileList := GetAppFileList(appListV2) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) - testDirMC := "c3appfw-mc-" + RandomDNSName(4) - uploadedFiles, err := backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + RandomDNSName(3) - appFrameworkSpecMC := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, testDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) - testDirIdxc := "c3appfw-idxc-" + RandomDNSName(4) - uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - // Upload V2 apps for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) - testDirShc := "c3appfw-shc-" + RandomDNSName(4) - uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + RandomDNSName(3) - appFrameworkSpecIdxc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, testDirIdxc, 60) - appFrameworkSpecShc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, testDirShc, 60) - - resourceVersion := GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - ClusterMasterReady(ctx, deployment, testcaseEnvInst) - SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - splunkPodAge := GetPodsStartTime(testcaseEnvInst.GetName()) - - // Initial verifications - idxcPodNames := GeneratePodNameSlice(IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := GeneratePodNameSlice(SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Downgrade apps - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) - backend.DeleteFiles(ctx, *uploadedApps) - *uploadedApps = nil - - resourceVersion = GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - appVersion = "V1" - appFileList = GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexers", appVersion)) - uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexers", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) - uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Monitoring Console", appVersion)) - uploadedFiles, err = backend.UploadFiles(ctx, testDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Monitoring Console", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - ClusterMasterReady(ctx, deployment, testcaseEnvInst) - SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - splunkPodAge = GetPodsStartTime(testcaseEnvInst.GetName()) - - // Final verifications - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = GetAppFileList(appListV1) - allAppSourceInfo = []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) -} - -// C3AppFrameworkLocalScopeUpgradeTest deploys a C3 SVA with App Framework -// enabled using local scope, installs V1 apps, then upgrades them to V2. -func C3AppFrameworkLocalScopeUpgradeTest( - ctx context.Context, - backend CloudStorageBackend, - testcaseEnvInst *TestCaseEnv, - deployment *Deployment, - appListV1, appListV2 []string, - downloadDirV1, downloadDirV2 string, - uploadedApps *[]string, -) { - appVersion := "V1" - appFileList := GetAppFileList(appListV1) - - // Upload V1 apps for Indexer Cluster - testDirIdxc := "c3appfw-idxc-" + RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Indexer Cluster", appVersion)) - uploadedFiles, err := backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - // Upload V1 apps for Search Head Cluster - testDirShc := "c3appfw-shc-" + RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps for Search Head Cluster", appVersion)) - uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeLocal + RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeLocal + RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + RandomDNSName(3) - appFrameworkSpecIdxc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, testDirIdxc, 60) - appFrameworkSpecShc := GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, testDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - ClusterMasterReady(ctx, deployment, testcaseEnvInst) - SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - splunkPodAge := GetPodsStartTime(testcaseEnvInst.GetName()) - - // Initial verifications - idxcPodNames := GeneratePodNameSlice(IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := GeneratePodNameSlice(SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(DeployerPod, deployment.GetName())} - cmAppSourceInfo := AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Upgrade apps - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on cloud storage", appVersion)) - backend.DeleteFiles(ctx, *uploadedApps) - *uploadedApps = nil - - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps", appVersion)) - appFileList = GetAppFileList(appListV2) - uploadedFiles, err = backend.UploadFiles(ctx, testDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Indexer Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - uploadedFiles, err = backend.UploadFiles(ctx, testDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for Search Head Cluster", appVersion)) - *uploadedApps = append(*uploadedApps, uploadedFiles...) - - WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - ClusterMasterReady(ctx, deployment, testcaseEnvInst) - SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - splunkPodAge = GetPodsStartTime(testcaseEnvInst.GetName()) - - // Upgrade verifications - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = GetAppFileList(appListV2) - allAppSourceInfo = []AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) -} diff --git a/test/testenv/cloud_storage_backend.go b/test/testenv/cloud_storage_backend.go index b25587b67..633ffeb8c 100644 --- a/test/testenv/cloud_storage_backend.go +++ b/test/testenv/cloud_storage_backend.go @@ -75,7 +75,8 @@ func (b *AzureBackend) DeleteFiles(ctx context.Context, uploadedFiles []string) } func (b *AzureBackend) DownloadFiles(ctx context.Context, remoteDir string, localDir string, fileList []string) error { - return DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, localDir, remoteDir, fileList) + containerName := "/test-data/" + remoteDir + return DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, localDir, containerName, fileList) } func (b *AzureBackend) GetCloudProvider() string { return "azure" } diff --git a/test/testenv/s3utils.go b/test/testenv/s3utils.go index a7309fd30..6ec98714e 100644 --- a/test/testenv/s3utils.go +++ b/test/testenv/s3utils.go @@ -21,7 +21,7 @@ import ( // Set Global Variables var ( - ClusterProvider = os.Getenv("TEST_CLUSTER_PLATFORM") + ClusterProvider = os.Getenv("CLUSTER_PROVIDER") ) // Set S3 Variables From c20d76abdb07d7e1f71c6c6df3359f7e748fea4c Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Mon, 16 Mar 2026 13:43:31 +0100 Subject: [PATCH 04/12] Trigger GCP and Azure pipelines --- .github/workflows/int-test-azure-workflow.yml | 1 + .github/workflows/int-test-gcp-workflow.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/int-test-azure-workflow.yml b/.github/workflows/int-test-azure-workflow.yml index 172983ed4..be4d31f41 100644 --- a/.github/workflows/int-test-azure-workflow.yml +++ b/.github/workflows/int-test-azure-workflow.yml @@ -9,6 +9,7 @@ on: branches: - develop - main + - CSPL-4601 paths-ignore: - 'docs/**' - '*.md' diff --git a/.github/workflows/int-test-gcp-workflow.yml b/.github/workflows/int-test-gcp-workflow.yml index 24a266de7..7d38925cc 100644 --- a/.github/workflows/int-test-gcp-workflow.yml +++ b/.github/workflows/int-test-gcp-workflow.yml @@ -9,6 +9,7 @@ on: branches: - develop - main + - CSPL-4601 paths-ignore: - 'docs/**' - '*.md' From d118b9c4ace6ffffaaf3e59daf83e960f33a153d Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Mon, 16 Mar 2026 15:30:20 +0100 Subject: [PATCH 05/12] fix gcp pipeline --- .github/workflows/int-test-gcp-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/int-test-gcp-workflow.yml b/.github/workflows/int-test-gcp-workflow.yml index 7d38925cc..b35e0aeb0 100644 --- a/.github/workflows/int-test-gcp-workflow.yml +++ b/.github/workflows/int-test-gcp-workflow.yml @@ -245,7 +245,7 @@ jobs: - name: Setup Kustomize run: | - curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash + curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- 5.4.3 sudo mv kustomize /usr/local/bin/ - name: Verify kubectl Configuration From 3827767a4b6e9cc4a2b7c76e65a1596c7f360572 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Mon, 16 Mar 2026 17:06:39 +0100 Subject: [PATCH 06/12] udpate test matrix - WIP --- .github/workflows/int-test-azure-workflow.yml | 2 +- .github/workflows/int-test-gcp-workflow.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/int-test-azure-workflow.yml b/.github/workflows/int-test-azure-workflow.yml index be4d31f41..b0b8aca31 100644 --- a/.github/workflows/int-test-azure-workflow.yml +++ b/.github/workflows/int-test-azure-workflow.yml @@ -117,7 +117,7 @@ jobs: SPLUNK_ENTERPRISE_RELEASE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_RELEASE_IMAGE }} SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator - TEST_FOCUS: azure_sanity + TEST_FOCUS: appframework # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" CLUSTER_PROVIDER: azure diff --git a/.github/workflows/int-test-gcp-workflow.yml b/.github/workflows/int-test-gcp-workflow.yml index b35e0aeb0..e85984581 100644 --- a/.github/workflows/int-test-gcp-workflow.yml +++ b/.github/workflows/int-test-gcp-workflow.yml @@ -71,11 +71,11 @@ jobs: strategy: matrix: test_focus: - - { order: 1, name: "c3_gcp_sanity" } - - { order: 2, name: "c3_mgr_gcp_sanity" } - - { order: 3, name: "m4_gcp_sanity" } - - { order: 4, name: "m4_mgr_gcp_sanity" } - - { order: 5, name: "s1_gcp_sanity" } + - { order: 1, name: "masterappframeworkc3" } + - { order: 2, name: "managerappframeworkc3" } + - { order: 3, name: "masterappframeworkm4" } + - { order: 4, name: "managerappframeworkm4" } + - { order: 5, name: "appframeworksS1" } runs-on: ubuntu-latest needs: build-operator-image env: From eceb1b24de8d0c31bfe46bd712e6c59c20817d72 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Mon, 16 Mar 2026 17:55:18 +0100 Subject: [PATCH 07/12] GenerateAppFrameworkVolumeSpec --- test/appframework/c3/appframework_test.go | 12 +++++------ .../c3/manager_appframework_test.go | 12 +++++------ test/appframework/m4/appframework_test.go | 4 ++-- .../m4/manager_appframework_test.go | 4 ++-- test/testenv/appframework_utils.go | 20 +++++++++++++++++++ 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/test/appframework/c3/appframework_test.go b/test/appframework/c3/appframework_test.go index b8a7824d5..bd4b677f7 100644 --- a/test/appframework/c3/appframework_test.go +++ b/test/appframework/c3/appframework_test.go @@ -601,7 +601,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -613,7 +613,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -809,7 +809,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -820,7 +820,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -1537,7 +1537,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1548,7 +1548,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, diff --git a/test/appframework/c3/manager_appframework_test.go b/test/appframework/c3/manager_appframework_test.go index 7d63be517..996062c14 100644 --- a/test/appframework/c3/manager_appframework_test.go +++ b/test/appframework/c3/manager_appframework_test.go @@ -1153,7 +1153,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1165,7 +1165,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -1361,7 +1361,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1372,7 +1372,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -2089,7 +2089,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -2100,7 +2100,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, diff --git a/test/appframework/m4/appframework_test.go b/test/appframework/m4/appframework_test.go index 3c96bee17..3208f3343 100644 --- a/test/appframework/m4/appframework_test.go +++ b/test/appframework/m4/appframework_test.go @@ -1436,7 +1436,7 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Cluster master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1447,7 +1447,7 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ diff --git a/test/appframework/m4/manager_appframework_test.go b/test/appframework/m4/manager_appframework_test.go index 1bbba550b..25a0bd25b 100644 --- a/test/appframework/m4/manager_appframework_test.go +++ b/test/appframework/m4/manager_appframework_test.go @@ -1435,7 +1435,7 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, testDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1446,7 +1446,7 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, testDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index e9879679b..359ca0844 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -427,6 +427,26 @@ func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, return appFrameworkSpec } +// GenerateAppFrameworkVolumeSpec returns a VolumeSpec appropriate for the current ClusterProvider. +// Use this instead of calling GenerateIndexVolumeSpec directly with hardcoded provider values. +func GenerateAppFrameworkVolumeSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeName string) enterpriseApi.VolumeSpec { + switch ClusterProvider { + case "eks": + return GenerateIndexVolumeSpec(volumeName, GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3", GetDefaultS3Region()) + case "azure": + managedID := os.Getenv("AZURE_MANAGED_ID_ENABLED") + if managedID == "false" { + return GenerateIndexVolumeSpecAzure(volumeName, GetAzureEndpoint(ctx), testenvInstance.GetIndexSecretName(), "azure", "blob") + } + return GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob") + case "gcp": + return GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultS3Region()) + default: + testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") + return enterpriseApi.VolumeSpec{} + } +} + // WaitforPhaseChange Wait for 2 mins or when phase change on is seen on a CR for any particular app func WaitforPhaseChange(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string, appSourceName string, appList []string) { startTime := time.Now() From c542650bca9779305b15f4d2e215aa7f042d2db8 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Mon, 16 Mar 2026 22:22:27 +0100 Subject: [PATCH 08/12] Add region --- test/testenv/appframework_utils.go | 4 ++-- test/testenv/remote_index_utils.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index 359ca0844..da46ddd27 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -401,7 +401,7 @@ func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob")} } case "gcp": - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultS3Region())} + volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultGCPRegion())} default: testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") @@ -440,7 +440,7 @@ func GenerateAppFrameworkVolumeSpec(ctx context.Context, testenvInstance *TestCa } return GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob") case "gcp": - return GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultS3Region()) + return GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultGCPRegion()) default: testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") return enterpriseApi.VolumeSpec{} diff --git a/test/testenv/remote_index_utils.go b/test/testenv/remote_index_utils.go index 551a377d0..c46642b75 100644 --- a/test/testenv/remote_index_utils.go +++ b/test/testenv/remote_index_utils.go @@ -116,6 +116,7 @@ func GenerateIndexVolumeSpecAzure(volumeName string, endpoint string, secretRef SecretRef: secretRef, Provider: provider, Type: storageType, + Region: Region, } } @@ -127,6 +128,7 @@ func GenerateIndexVolumeSpecAzureManagedID(volumeName string, endpoint string, p Path: azureIndexesContainer, Provider: provider, Type: storageType, + Region: Region, } } From e7d874a21617240dd98bfdc020efbc417bcfe00e Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Wed, 18 Mar 2026 10:28:06 +0100 Subject: [PATCH 09/12] use gke auth plugin --- .github/workflows/int-test-gcp-workflow.yml | 5 ++++- test/testenv/deployment.go | 14 ++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/int-test-gcp-workflow.yml b/.github/workflows/int-test-gcp-workflow.yml index e85984581..08f5babc6 100644 --- a/.github/workflows/int-test-gcp-workflow.yml +++ b/.github/workflows/int-test-gcp-workflow.yml @@ -136,11 +136,12 @@ jobs: uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - install_components: 'kubectl' + install_components: 'kubectl,gke-gcloud-auth-plugin' - name: Set GCP Project run: | gcloud config set project ${{ env.GCP_PROJECT_ID }} + export USE_GKE_GCLOUD_AUTH_PLUGIN=True - name: Create GKE Cluster run: | @@ -150,6 +151,7 @@ jobs: - name: Get Kubernetes Credentials run: | + export USE_GKE_GCLOUD_AUTH_PLUGIN=True gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }} - name: Allow Pulling from Artifact Registry @@ -226,6 +228,7 @@ jobs: - name: Get Kubernetes Credentials run: | + export USE_GKE_GCLOUD_AUTH_PLUGIN=True gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }} - name: Get GKE Credentials diff --git a/test/testenv/deployment.go b/test/testenv/deployment.go index e639a9513..6b4440981 100644 --- a/test/testenv/deployment.go +++ b/test/testenv/deployment.go @@ -216,8 +216,11 @@ func (d *Deployment) PodExecCommand(ctx context.Context, podName string, cmd []s if err != nil { return "", "", err } - //FIXME - restClient, err := apiutil.RESTClientForGVK(gvk, false, restConfig, serializer.NewCodecFactory(scheme.Scheme), http.DefaultClient) + httpClient, err := rest.HTTPClientFor(restConfig) + if err != nil { + return "", "", err + } + restClient, err := apiutil.RESTClientForGVK(gvk, false, restConfig, serializer.NewCodecFactory(scheme.Scheme), httpClient) if err != nil { return "", "", err } @@ -263,8 +266,11 @@ func (d *Deployment) OperatorPodExecCommand(ctx context.Context, podName string, if err != nil { return "", "", err } - //FIXME - restClient, err := apiutil.RESTClientForGVK(gvk, false, restConfig, serializer.NewCodecFactory(scheme.Scheme), http.DefaultClient) + httpClient, err := rest.HTTPClientFor(restConfig) + if err != nil { + return "", "", err + } + restClient, err := apiutil.RESTClientForGVK(gvk, false, restConfig, serializer.NewCodecFactory(scheme.Scheme), httpClient) if err != nil { return "", "", err } From 61239291851001e9a397fa720fdf8b5133ff7b00 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Mon, 23 Mar 2026 14:40:04 +0100 Subject: [PATCH 10/12] fix --- .github/workflows/int-test-azure-workflow.yml | 40 +++++++++---------- .github/workflows/int-test-gcp-workflow.yml | 18 ++------- .../c3/appframework_suite_test.go | 2 +- test/appframework/c3/appframework_test.go | 6 +-- .../c3/manager_appframework_test.go | 6 +-- .../m4/appframework_suite_test.go | 2 +- test/appframework/m4/appframework_test.go | 6 +-- .../m4/manager_appframework_test.go | 6 +-- .../s1/appframework_suite_test.go | 2 +- test/appframework/s1/appframework_test.go | 6 +-- test/trigger-tests.sh | 11 ++--- 11 files changed, 44 insertions(+), 61 deletions(-) diff --git a/.github/workflows/int-test-azure-workflow.yml b/.github/workflows/int-test-azure-workflow.yml index b0b8aca31..ec08f19a4 100644 --- a/.github/workflows/int-test-azure-workflow.yml +++ b/.github/workflows/int-test-azure-workflow.yml @@ -108,6 +108,17 @@ jobs: inlineScript: | az aks update -n ${{ env.TEST_CLUSTER_NAME }} -g ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} --attach-acr ${{ secrets.AZURE_CONTAINER_REGISTRY }} int-tests-appframeworkazure: + strategy: + fail-fast: false + matrix: + test: + [ + masterappframeworkc3, + managerappframeworkc3, + masterappframeworkm4, + managerappframeworkm4, + appframeworksS1, + ] runs-on: ubuntu-latest needs: setup-aks-cluster env: @@ -117,7 +128,7 @@ jobs: SPLUNK_ENTERPRISE_RELEASE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_RELEASE_IMAGE }} SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator - TEST_FOCUS: appframework + TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" CLUSTER_PROVIDER: azure @@ -133,7 +144,6 @@ jobs: AZURE_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.AZURE_ACR_LOGIN_SERVER }} AZURE_REGION: ${{ secrets.AZURE_REGION }} CLUSTER_WIDE: "true" - # AZURE_MANAGED_ID_ENABLED: "${{ matrix.auth_method_managed_id }}" AZURE_MANAGED_ID_ENABLED: "false" steps: # Need this because apps are downloaded from S3. @@ -201,22 +211,12 @@ jobs: resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} cluster-name: ${{ env.TEST_CLUSTER_NAME }} - name: install metric server - uses: Azure/aks-set-context@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} - cluster-name: ${{ env.TEST_CLUSTER_NAME }} - inlineScript: | - curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml - kubectl replace --force -f components.yaml || kubectl apply -f components.yaml + run: | + curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + kubectl replace --force -f components.yaml || kubectl apply -f components.yaml - name: install k8s dashboard - uses: Azure/aks-set-context@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} - cluster-name: ${{ env.TEST_CLUSTER_NAME }} - inlineScript: | - kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml + run: | + kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml - name: Setup Kustomize run: | sudo snap install kustomize @@ -237,20 +237,20 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v6 with: - name: "splunk-pods-logs--artifacts-appframeworkazure" + name: "splunk-pods-logs--artifacts-${{ matrix.test }}" path: "/tmp/pod_logs/**" - name: Upload Integration Test Results if: always() uses: actions/upload-artifact@v6 with: - name: "test-report-integration-azure" + name: "test-report-integration-azure-${{ matrix.test }}" path: report-junit*.xml - name: Publish Integration Test Report uses: mikepenz/action-junit-report@v6 if: always() with: report_paths: 'report-junit*.xml' - check_name: 'Integration Test Results (Azure)' + check_name: 'Integration Test Results (Azure) - ${{ matrix.test }}' detailed_summary: true - name: Cleanup Test Case artifacts if: ${{ always() }} diff --git a/.github/workflows/int-test-gcp-workflow.yml b/.github/workflows/int-test-gcp-workflow.yml index 08f5babc6..4be7a7b5b 100644 --- a/.github/workflows/int-test-gcp-workflow.yml +++ b/.github/workflows/int-test-gcp-workflow.yml @@ -84,7 +84,7 @@ jobs: ARTIFACT_REGISTRY: ${{ secrets.GCP_ARTIFACT_REGISTRY }} GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} GCP_REGION: ${{ secrets.GCP_REGION }} - AWS_S3_REGION: ${{ secrets.GCP_REGION }} + GCP_STORAGE_REGION: ${{ secrets.GCP_REGION }} GCP_ZONE: ${{ secrets.GCP_ZONE }} GCP_NETWORK: default # Adjust if using a custom network GCP_SUBNETWORK: default # Adjust if using a custom subnetwork @@ -112,7 +112,7 @@ jobs: GCP_TEST_CONTAINER: ${{ secrets.GCP_TEST_CONTAINER}} GCP_INDEXES_CONTAINER: ${{ secrets.GCP_INDEXES_CONTAINER}} ECR_REPOSITORY: ${{ secrets.GCP_ARTIFACT_REGISTRY }} - GCP_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.AZURE_ACR_LOGIN_SERVER }} + GCP_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.GCP_ARTIFACT_REGISTRY }} steps: - name: Set Test Cluster Name run: | @@ -141,7 +141,7 @@ jobs: - name: Set GCP Project run: | gcloud config set project ${{ env.GCP_PROJECT_ID }} - export USE_GKE_GCLOUD_AUTH_PLUGIN=True + echo "USE_GKE_GCLOUD_AUTH_PLUGIN=True" >> $GITHUB_ENV - name: Create GKE Cluster run: | @@ -151,19 +151,12 @@ jobs: - name: Get Kubernetes Credentials run: | - export USE_GKE_GCLOUD_AUTH_PLUGIN=True gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }} - name: Allow Pulling from Artifact Registry run: | gcloud auth configure-docker ${{ secrets.GCP_ARTIFACT_REGISTRY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - install_components: 'kubectl' - - name: Change Splunk Enterprise Image on Main Branches if: github.ref == 'refs/heads/main' run: | @@ -226,11 +219,6 @@ jobs: docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.GCP_ARTIFACT_REGISTRY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} docker push ${{ secrets.GCP_ARTIFACT_REGISTRY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} - - name: Get Kubernetes Credentials - run: | - export USE_GKE_GCLOUD_AUTH_PLUGIN=True - gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }} - - name: Get GKE Credentials uses: google-github-actions/get-gke-credentials@v1 with: diff --git a/test/appframework/c3/appframework_suite_test.go b/test/appframework/c3/appframework_suite_test.go index 86242b8db..c5a961317 100644 --- a/test/appframework/c3/appframework_suite_test.go +++ b/test/appframework/c3/appframework_suite_test.go @@ -57,7 +57,7 @@ var ( func TestBasic(t *testing.T) { if err := loadEnvFile(); err != nil { - panic("Error loading .env file: " + err.Error()) + t.Fatalf("Error loading .env file: %s", err.Error()) } RegisterFailHandler(Fail) diff --git a/test/appframework/c3/appframework_test.go b/test/appframework/c3/appframework_test.go index bd4b677f7..bb292d94f 100644 --- a/test/appframework/c3/appframework_test.go +++ b/test/appframework/c3/appframework_test.go @@ -67,7 +67,7 @@ var _ = Describe("c3appfw test", func() { AfterEach(func() { // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil && types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { testcaseEnvInst.SkipTeardown = true } if deployment != nil { @@ -79,11 +79,11 @@ var _ = Describe("c3appfw test", func() { } // Delete files uploaded to cloud storage - if !testcaseEnvInst.SkipTeardown { + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown { backend.DeleteFiles(ctx, uploadedApps) } - if filePresentOnOperator { + if filePresentOnOperator && testcaseEnvInst != nil { //Delete files from app-directory opPod := testenv.GetOperatorPodName(testcaseEnvInst) podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") diff --git a/test/appframework/c3/manager_appframework_test.go b/test/appframework/c3/manager_appframework_test.go index 996062c14..f601eea76 100644 --- a/test/appframework/c3/manager_appframework_test.go +++ b/test/appframework/c3/manager_appframework_test.go @@ -62,7 +62,7 @@ var _ = Describe("c3appfw test", func() { AfterEach(func() { // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil && types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { testcaseEnvInst.SkipTeardown = true } if deployment != nil { @@ -74,11 +74,11 @@ var _ = Describe("c3appfw test", func() { } // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown { backend.DeleteFiles(ctx, uploadedApps) } - if filePresentOnOperator { + if filePresentOnOperator && testcaseEnvInst != nil { //Delete files from app-directory opPod := testenv.GetOperatorPodName(testcaseEnvInst) podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") diff --git a/test/appframework/m4/appframework_suite_test.go b/test/appframework/m4/appframework_suite_test.go index dfe7c74b3..1cc496a19 100644 --- a/test/appframework/m4/appframework_suite_test.go +++ b/test/appframework/m4/appframework_suite_test.go @@ -57,7 +57,7 @@ var ( func TestBasic(t *testing.T) { if err := loadEnvFile(); err != nil { - panic("Error loading .env file: " + err.Error()) + t.Fatalf("Error loading .env file: %s", err.Error()) } RegisterFailHandler(Fail) diff --git a/test/appframework/m4/appframework_test.go b/test/appframework/m4/appframework_test.go index 3208f3343..0f47b1c73 100644 --- a/test/appframework/m4/appframework_test.go +++ b/test/appframework/m4/appframework_test.go @@ -67,21 +67,21 @@ var _ = Describe("m4appfw test", func() { AfterEach(func() { // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil && types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { testcaseEnvInst.SkipTeardown = true } if deployment != nil { deployment.Teardown() } // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown { backend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) } - if filePresentOnOperator { + if filePresentOnOperator && testcaseEnvInst != nil { //Delete files from app-directory opPod := testenv.GetOperatorPodName(testcaseEnvInst) podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") diff --git a/test/appframework/m4/manager_appframework_test.go b/test/appframework/m4/manager_appframework_test.go index 25a0bd25b..59ab6527f 100644 --- a/test/appframework/m4/manager_appframework_test.go +++ b/test/appframework/m4/manager_appframework_test.go @@ -66,21 +66,21 @@ var _ = Describe("m4appfw test", func() { AfterEach(func() { // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil && types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { testcaseEnvInst.SkipTeardown = true } if deployment != nil { deployment.Teardown() } // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown { backend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) } - if filePresentOnOperator { + if filePresentOnOperator && testcaseEnvInst != nil { //Delete files from app-directory opPod := testenv.GetOperatorPodName(testcaseEnvInst) podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") diff --git a/test/appframework/s1/appframework_suite_test.go b/test/appframework/s1/appframework_suite_test.go index 7093ff41c..a62a8c142 100644 --- a/test/appframework/s1/appframework_suite_test.go +++ b/test/appframework/s1/appframework_suite_test.go @@ -57,7 +57,7 @@ var ( func TestBasic(t *testing.T) { if err := loadEnvFile(); err != nil { - panic("Error loading .env file: " + err.Error()) + t.Fatalf("Error loading .env file: %s", err.Error()) } RegisterFailHandler(Fail) diff --git a/test/appframework/s1/appframework_test.go b/test/appframework/s1/appframework_test.go index b725f8805..04c9c1b72 100644 --- a/test/appframework/s1/appframework_test.go +++ b/test/appframework/s1/appframework_test.go @@ -57,21 +57,21 @@ var _ = Describe("s1appfw test", func() { AfterEach(func() { // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil && types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { testcaseEnvInst.SkipTeardown = true } if deployment != nil { deployment.Teardown() } // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown { backend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) } - if filePresentOnOperator { + if filePresentOnOperator && testcaseEnvInst != nil { //Delete files from app-directory opPod := testenv.GetOperatorPodName(testcaseEnvInst) podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") diff --git a/test/trigger-tests.sh b/test/trigger-tests.sh index dce729e03..415ed3b19 100644 --- a/test/trigger-tests.sh +++ b/test/trigger-tests.sh @@ -100,12 +100,12 @@ case ${CLUSTER_PROVIDER} in fi ;; "gcp") - if [[ -z "${GCP_ENTERPRISE_LICENSE_LOCATION}" ]]; then - echo "License path not set. Changing to default" + if [[ -z "${ENTERPRISE_LICENSE_LOCATION}" ]]; then + echo "License path not set. Changing to GCP default" export ENTERPRISE_LICENSE_LOCATION="${GCP_ENTERPRISE_LICENSE_LOCATION}" fi if [[ -z "${ENTERPRISE_LICENSE_LOCATION}" ]]; then - echo "License path not set. Changing to default" + echo "License path not set. Falling back to S3 path" export ENTERPRISE_LICENSE_LOCATION="${ENTERPRISE_LICENSE_S3_PATH}" fi @@ -118,11 +118,6 @@ case ${CLUSTER_PROVIDER} in echo "Test bucket not set. Changing to default" export TEST_INDEXES_S3_BUCKET="${INDEXES_S3_BUCKET}" fi - - if [[ -z "${S3_REGION}" ]]; then - echo "S3 Region not set. Changing to default" - export S3_REGION="${AWS_S3_REGION}" - fi ;; esac From 5a52dfdf5e8ca615f68eeee6bbd354a50553e9aa Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Tue, 24 Mar 2026 11:29:41 +0100 Subject: [PATCH 11/12] Setup SA --- .github/workflows/int-test-azure-workflow.yml | 43 ++++++++++++++--- .github/workflows/int-test-gcp-workflow.yml | 19 -------- test/deploy-eks-cluster.sh | 2 - test/deploy-gcp-cluster.sh | 47 ++++++++++++++++++- 4 files changed, 82 insertions(+), 29 deletions(-) diff --git a/.github/workflows/int-test-azure-workflow.yml b/.github/workflows/int-test-azure-workflow.yml index ec08f19a4..16c90739a 100644 --- a/.github/workflows/int-test-azure-workflow.yml +++ b/.github/workflows/int-test-azure-workflow.yml @@ -204,12 +204,43 @@ jobs: run: | docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.AZURE_ACR_LOGIN_SERVER }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} docker push ${{ secrets.AZURE_ACR_LOGIN_SERVER }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} - - name: Azure Kubernetes set context - uses: Azure/aks-set-context@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} - cluster-name: ${{ env.TEST_CLUSTER_NAME }} + - name: Get AKS credentials + run: | + az aks get-credentials --resource-group ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} --name ${{ env.TEST_CLUSTER_NAME }} --admin --overwrite-existing + - name: Setup long-lived service account auth + run: | + kubectl create serviceaccount ci-test-runner -n kube-system 2>/dev/null || true + kubectl create clusterrolebinding ci-test-runner-admin \ + --clusterrole=cluster-admin \ + --serviceaccount=kube-system:ci-test-runner 2>/dev/null || true + + cat <<'EOF' | kubectl apply -f - + apiVersion: v1 + kind: Secret + metadata: + name: ci-test-runner-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: ci-test-runner + type: kubernetes.io/service-account-token + EOF + + echo "Waiting for service account token..." + TOKEN="" + for i in $(seq 1 30); do + TOKEN=$(kubectl get secret ci-test-runner-token -n kube-system -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null) + if [ -n "${TOKEN}" ]; then break; fi + sleep 2 + done + + if [ -n "${TOKEN}" ]; then + kubectl config set-credentials ci-test-runner --token="${TOKEN}" + kubectl config set-context --current --user=ci-test-runner + echo "Switched kubeconfig to long-lived service account token" + else + echo "WARNING: Could not create SA token, keeping admin cert auth" + fi + kubectl cluster-info - name: install metric server run: | curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml diff --git a/.github/workflows/int-test-gcp-workflow.yml b/.github/workflows/int-test-gcp-workflow.yml index 4be7a7b5b..0ac41c37d 100644 --- a/.github/workflows/int-test-gcp-workflow.yml +++ b/.github/workflows/int-test-gcp-workflow.yml @@ -149,10 +149,6 @@ jobs: export GKE_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} make cluster-up - - name: Get Kubernetes Credentials - run: | - gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }} - - name: Allow Pulling from Artifact Registry run: | gcloud auth configure-docker ${{ secrets.GCP_ARTIFACT_REGISTRY }} @@ -162,15 +158,6 @@ jobs: run: | echo "SPLUNK_ENTERPRISE_IMAGE=${{ steps.dotenv.outputs.SPLUNK_ENTERPRISE_RELEASE_IMAGE }}" >> $GITHUB_ENV - - name: Authenticate to GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} - - - name: Set GCP Project - run: | - gcloud config set project ${{ env.GCP_PROJECT_ID }} - - name: Install Kubectl uses: azure/setup-kubectl@v3 with: @@ -219,12 +206,6 @@ jobs: docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.GCP_ARTIFACT_REGISTRY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} docker push ${{ secrets.GCP_ARTIFACT_REGISTRY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} - - name: Get GKE Credentials - uses: google-github-actions/get-gke-credentials@v1 - with: - cluster_name: ${{ env.CLUSTER_NAME }} - location: ${{ env.GCP_ZONE }} - - name: Install Metrics Server run: | curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml diff --git a/test/deploy-eks-cluster.sh b/test/deploy-eks-cluster.sh index cf30ba81e..aac0e46fa 100755 --- a/test/deploy-eks-cluster.sh +++ b/test/deploy-eks-cluster.sh @@ -140,8 +140,6 @@ function createCluster() { return 1 fi - - # Login to ECR registry so images can be push and pull from later whe # Output echo "EKS cluster nodes:" eksctl get cluster --name=${TEST_CLUSTER_NAME} diff --git a/test/deploy-gcp-cluster.sh b/test/deploy-gcp-cluster.sh index ceb7ba515..b494d1b9e 100755 --- a/test/deploy-gcp-cluster.sh +++ b/test/deploy-gcp-cluster.sh @@ -20,6 +20,47 @@ if [[ -z "${GKE_CLUSTER_K8_VERSION}" ]]; then export GKE_CLUSTER_K8_VERSION="1.29" fi +function setupServiceAccountAuth() { + echo "Creating long-lived service account for CI test runs..." + + kubectl create serviceaccount ci-test-runner -n kube-system 2>/dev/null || true + kubectl create clusterrolebinding ci-test-runner-admin \ + --clusterrole=cluster-admin \ + --serviceaccount=kube-system:ci-test-runner 2>/dev/null || true + + cat </dev/null | base64 -d 2>/dev/null) + if [ -n "${TOKEN}" ]; then + break + fi + sleep 2 + done + + if [ -z "${TOKEN}" ]; then + echo "WARNING: Failed to obtain service account token, keeping exec-based auth" + return 0 + fi + + kubectl config set-credentials ci-test-runner --token="${TOKEN}" + kubectl config set-context --current --user=ci-test-runner + + echo "Switched kubeconfig to long-lived service account token" + kubectl cluster-info +} + function deleteCluster() { echo "Cleanup remaining PVC on the GKE Cluster ${TEST_CLUSTER_NAME}" tools/cleanup.sh @@ -49,7 +90,8 @@ function createCluster() { --subnetwork ${GCP_SUBNETWORK} \ --machine-type n2-standard-8 \ --scopes "https://www.googleapis.com/auth/cloud-platform" \ - --enable-ip-alias + --enable-ip-alias \ + --cluster-version=${GKE_CLUSTER_K8_VERSION} if [ $? -ne 0 ]; then echo "Unable to create cluster - ${TEST_CLUSTER_NAME}" return 1 @@ -67,7 +109,8 @@ function createCluster() { return 1 fi - # Output + setupServiceAccountAuth + echo "GKE cluster nodes:" kubectl get nodes } From 4ce0123f0b8116f9d1be0a73e0e0ecd4c950b9e1 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Wed, 25 Mar 2026 12:05:58 +0100 Subject: [PATCH 12/12] version check with eventually --- test/testenv/verificationutils.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index cb611254d..ce58ee8a9 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -22,10 +22,11 @@ import ( "fmt" "math/rand" "os/exec" - "sigs.k8s.io/controller-runtime/pkg/client" "strings" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + gomega "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -924,7 +925,14 @@ func VerifyAppInstalled(ctx context.Context, deployment *Deployment, testenvInst expectedVersion = AppInfo[appName]["V1"] } testenvInstance.Log.Info("Verify app", "On pod", podName, "App name", appName, "Expected version", expectedVersion, "Version installed", versionInstalled, "Updated", checkupdated) - gomega.Expect(versionInstalled).Should(gomega.Equal(expectedVersion)) + gomega.Eventually(func() string { + _, ver, err := GetPodAppStatus(ctx, deployment, podName, ns, appName, clusterWideInstall) + if err != nil { + testenvInstance.Log.Info("Retrying app version check", "On pod", podName, "App name", appName, "Error", err) + return "" + } + return ver + }, 5*time.Minute, PollInterval).Should(gomega.Equal(expectedVersion)) } } }