diff --git a/test/c3/appframework/appframework_test.go b/test/c3/appframework/appframework_test.go index b239d13fe..c754b8c36 100644 --- a/test/c3/appframework/appframework_test.go +++ b/test/c3/appframework/appframework_test.go @@ -36,14 +36,6 @@ var _ = Describe("c3appfw test", func() { var err error deployment, err = testenvInstance.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") - - // Upload V1 apps to S3 - s3TestDir = "c3appfw-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1, 1) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - }) AfterEach(func() { @@ -128,7 +120,13 @@ var _ = Describe("c3appfw test", func() { // Verify MC is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Create App framework Spec for C3 + // Upload V1 apps to S3 + s3TestDir = "c3appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + uploadedApps = append(uploadedApps, uploadedFiles...) + + // Create App framework Spec volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} @@ -292,11 +290,6 @@ var _ = Describe("c3appfw test", func() { * Verify apps are copied, installed and downgraded on MC AND SH,Indexers pods */ - // Delete pre-installed apps on S3 - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - uploadedApps = nil - testenvInstance.Log.Info("Testing downgrade scenario") - // Upload newer version of apps to S3 s3TestDir = "c3appfw-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV2, 2) @@ -414,8 +407,6 @@ var _ = Describe("c3appfw test", func() { // Verify bundle push status testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) - // Saving current V1 bundle hash for future comparision - clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment) // Verify Apps are copied to location allPodNames := testenv.DumpGetPods(testenvInstance.GetName()) @@ -492,7 +483,7 @@ var _ = Describe("c3appfw test", func() { testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledIndexerReplicas), "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledSHReplicas)) // Saving current V2 bundle hash to future comparision with new config bundle hash - clusterManagerBundleHash = testenv.GetClusterManagerBundleHash(deployment) + clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment) // Verify Apps are copied to location allPodNames = testenv.DumpGetPods(testenvInstance.GetName()) @@ -565,6 +556,13 @@ var _ = Describe("c3appfw test", func() { Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { It("smoke, c3, appframework: can deploy a C3 SVA and have apps installed locally on CM and SHC Deployer", func() { + // Upload V1 apps to S3 + s3TestDir = "c3appfw-" + testenv.RandomDNSName(4) + appFileList := testenv.GetAppFileList(appListV1, 1) + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + uploadedApps = append(uploadedApps, uploadedFiles...) + // Create App framework Spec // volumeSpec: Volume name, Endpoint, Path and SecretRef volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) @@ -591,7 +589,7 @@ var _ = Describe("c3appfw test", func() { // Create Single site Cluster and SHC, with App Framework enabled on CM and SHC Deployer indexerReplicas := 3 testenvInstance.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpec, 10, false) + err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpec, 10, false) Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") // Ensure that the CM goes to Ready phase @@ -609,7 +607,6 @@ var _ = Describe("c3appfw test", func() { // Verify Apps are downloaded by init-container initContDownloadLocation := "/init-apps/" + appSourceName podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - appFileList := testenv.GetAppFileList(appListV1, 1) appVersion := "V1" testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList) testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) @@ -636,7 +633,7 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testenvInstance.Log.Info("Uploading apps S3 for", "version", appVersion) appFileList = testenv.GetAppFileList(appListV2, 2) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -676,12 +673,9 @@ var _ = Describe("c3appfw test", func() { Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA and have ES app installed on SHC", func() { - // Delete apps on S3 for new Apps - testenvInstance.Log.Info("Delete existing apps on S3 before starting upload of ES App") - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - uploadedApps = nil + // Create local directory for file download + s3TestDir = "c3appfw-" + testenv.RandomDNSName(4) - // ES is a huge file, we configure it here rather than in BeforeSuite/BeforeEach to save time for other tests // Upload ES app to S3 esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp, 1) @@ -768,6 +762,9 @@ var _ = Describe("c3appfw test", func() { testenvInstance.Log.Info("Verfiy ES app is installed locally on deployer pod") testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), deployerPod, esApp, true, "disabled", false, false) + // Verify bundle push status + testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(shSpec.Replicas)) + // Verify apps are installed on SHs testenvInstance.Log.Info("Verfiy ES app is installed on Search Heads") podNames := []string{} @@ -782,9 +779,8 @@ var _ = Describe("c3appfw test", func() { Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on CM and SHC Deployer, and cluster-wide on Peers and SHs", func() { - // Delete apps on S3 for new Apps to split them across both cluster and local - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - uploadedApps = nil + // Create directory for file download + s3TestDir = "c3appfw-" + testenv.RandomDNSName(4) // Split Applist into 2 list for local and cluster install appListLocal := appListV1[len(appListV1)/2:] @@ -1013,16 +1009,23 @@ var _ = Describe("c3appfw test", func() { Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { It("smoke, c3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { + // Create directory for app file download + s3TestDir = "c3appfw-" + testenv.RandomDNSName(4) + // Creating a bigger list of apps to be installed than the default one - appList := append(appListV1, "splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials") + appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} appFileList := testenv.GetAppFileList(appList, 1) // Download App from S3 - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, testenv.GetAppFileList(appList, 1)) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") + // Create consolidated list of app files + appList = append(appListV1, appList...) + appFileList = testenv.GetAppFileList(appList, 1) + // Upload app to S3 - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, testenv.GetAppFileList(appList, 1), downloadDirV1) + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) diff --git a/test/m4/appframework/appframework_test.go b/test/m4/appframework/appframework_test.go index 3ccfdd48a..82ca388fe 100644 --- a/test/m4/appframework/appframework_test.go +++ b/test/m4/appframework/appframework_test.go @@ -38,13 +38,6 @@ var _ = Describe("m4appfw test", func() { deployment, err = testenvInstance.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") - // Upload V1 apps to S3 - s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1, 1) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - }) AfterEach(func() { @@ -121,7 +114,13 @@ var _ = Describe("m4appfw test", func() { // Verify MC is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Create App framework Spec for M4 + // Upload V1 apps to S3 + s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + uploadedApps = append(uploadedApps, uploadedFiles...) + + // Create App framework Spec volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -282,11 +281,6 @@ var _ = Describe("m4appfw test", func() { * Verify apps are copied and downgraded on MC and on SH and Indexers pods */ - // Delete pre-installed apps on S3 - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - uploadedApps = nil - testenvInstance.Log.Info("Testing downgrade scenario") - // Upload newer version of apps to S3 s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV2, 2) @@ -539,6 +533,12 @@ var _ = Describe("m4appfw test", func() { // Verify MC is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) + // Upload V1 apps to S3 + s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + uploadedApps = append(uploadedApps, uploadedFiles...) + // Create App framework Spec for M4 volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} @@ -744,6 +744,13 @@ var _ = Describe("m4appfw test", func() { Context("Clustered deployment (M4 - clustered indexer, search head cluster)", func() { It("integration, m4, appframework: can deploy a M4 SVA and have apps installed locally on CM and SHC Deployer", func() { + // Upload V1 apps to S3 + s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) + appFileList := testenv.GetAppFileList(appListV1, 1) + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + uploadedApps = append(uploadedApps, uploadedFiles...) + // Create App framework Spec // volumeSpec: Volume name, Endpoint, Path and SecretRef volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) @@ -771,7 +778,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 testenvInstance.Log.Info("Deploy Multisite Indexer Cluster") - err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpec, true, 10, "", "") + err = deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpec, true, 10, "", "") Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") // Ensure that the CM goes to Ready phase @@ -786,7 +793,6 @@ var _ = Describe("m4appfw test", func() { // Verify Apps are downloaded by init-container initContDownloadLocation := "/init-apps/" + appSourceName podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - appFileList := testenv.GetAppFileList(appListV1, 1) appVersion := "V1" testenvInstance.Log.Info("Verify Apps are downloaded by init container local install on CM and Deployer for apps", "version", appVersion) testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) @@ -811,7 +817,7 @@ var _ = Describe("m4appfw test", func() { //Upload new Versioned Apps to S3 appFileList = testenv.GetAppFileList(appListV2, 2) appVersion = "V2" - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) testenvInstance.Log.Info("Uploading apps S3 for", "version", appVersion) diff --git a/test/s1/appframework/appframework_test.go b/test/s1/appframework/appframework_test.go index 8af8a457a..7c9f653b2 100644 --- a/test/s1/appframework/appframework_test.go +++ b/test/s1/appframework/appframework_test.go @@ -37,14 +37,6 @@ var _ = Describe("s1appfw test", func() { var err error deployment, err = testenvInstance.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") - - // Upload V1 apps to S3 - s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1, 1) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - }) AfterEach(func() { @@ -87,6 +79,13 @@ var _ = Describe("s1appfw test", func() { mcName := deployment.GetName() + // Upload V1 apps to S3 + s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) + appFileList := testenv.GetAppFileList(appListV1, 1) + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + uploadedApps = append(uploadedApps, uploadedFiles...) + // Create App framework Spec for Standalone volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} @@ -131,8 +130,8 @@ var _ = Describe("s1appfw test", func() { // Upload Apps to S3 for MC s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1, 1) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + appFileList = testenv.GetAppFileList(appListV1, 1) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -321,16 +320,12 @@ var _ = Describe("s1appfw test", func() { }) }) - // Removing test from Nightly and Smoke runs due to consistent failure. Context("appframework Standalone deployment (S1) with App Framework", func() { It("s1, integration, appframework: can deploy a Standalone and have ES app installed", func() { - //Delete apps on S3 for new Apps - testenvInstance.Log.Info("Delete Apps on S3 before upload ES") - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - uploadedApps = nil + // Create local app Directory + s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) - // ES is a huge file, we configure it here rather than in BeforeSuite/BeforeEach to save time for other tests // Upload ES app to S3 esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp, 1) @@ -392,12 +387,14 @@ var _ = Describe("s1appfw test", func() { Context("appframework Standalone deployment (S1) with App Framework", func() { It("smoke, s1, appframework: can deploy a standalone instance with App Framework enabled and install around 350MB of apps at once", func() { + // Create local app Directory + s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) + // Creating a bigger list of apps to be installed than the default one appList := append(appListV1, testenv.RestartNeededApps...) - appFileList := testenv.GetAppFileList(appList, 1) // Download apps from S3 - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, testenv.GetAppFileList(appList, 1)) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, testenv.GetAppFileList(testenv.RestartNeededApps, 1)) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 @@ -444,7 +441,7 @@ var _ = Describe("s1appfw test", func() { // Verify apps are downloaded by init-container initContDownloadLocation := "/init-apps/" + appSourceName podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - appFileList = testenv.GetAppFileList(appListV1, 1) + appFileList := testenv.GetAppFileList(appListV1, 1) testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appFileList, initContDownloadLocation) // Verify apps are copied to location diff --git a/test/testenv/cmutil.go b/test/testenv/cmutil.go index cfb9c0de7..edeb5c30d 100644 --- a/test/testenv/cmutil.go +++ b/test/testenv/cmutil.go @@ -207,8 +207,12 @@ func RollHotBuckets(deployment *Deployment) bool { type ClusterMasterInfoEndpointResponse struct { Entry []struct { Content struct { - RollingRestartFlag bool `json:"rolling_restart_flag"` - ActiveBundle map[string]string `json:"active_bundle"` + RollingRestartFlag bool `json:"rolling_restart_flag"` + ActiveBundle struct { + BundlePath string `json:"bundle_path"` + Checksum string `json:"checksum"` + Timestamp int `json:"timestamp"` + } `json:"active_bundle"` } `json:"content"` } `json:"entry"` } @@ -281,7 +285,7 @@ func GetClusterManagerBundleHash(deployment *Deployment) string { podName := fmt.Sprintf(ClusterManagerPod, deployment.GetName()) restResponse := ClusterMasterInfoResponse(deployment, podName) - bundleHash := restResponse.Entry[0].Content.ActiveBundle["checksum"] + bundleHash := restResponse.Entry[0].Content.ActiveBundle.Checksum logf.Log.Info("Bundle Hash on Cluster Manager Found", "Hash", bundleHash) return bundleHash }