Skip to content

CSPL-1277: Check for status of bundle push on CM #536

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 20, 2021
Merged

CSPL-1277: Check for status of bundle push on CM #536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 74 additions & 22 deletions test/c3/appframework/appframework_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,19 @@ var _ = Describe("c3appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation)

// Get instance of current SHC CR with latest config
shcName := deployment.GetName() + "-shc"
shc := &enterpriseApi.SearchHeadCluster{}
err = deployment.GetInstance(shcName, shc)
shReplicas := int(shc.Spec.Replicas)
Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster")

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "")
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)
// Saving current V1 bundle hash for future comparision
clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment)

// Verify Apps are copied to location
allPodNames := testenv.DumpGetPods(testenvInstance.GetName())
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", "version", appVersion)
Expand Down Expand Up @@ -164,6 +177,10 @@ var _ = Describe("c3appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation)

// Verify bundle push status and compare bundle hash with previous v1 bundle hash
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash)
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)

// Verify Apps are copied to location
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", "version", appVersion)
testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, true)
Expand All @@ -176,18 +193,14 @@ var _ = Describe("c3appfw test", func() {
testenvInstance.Log.Info("Verify Apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion)
testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, "enabled", true, true)

// Get instance of current SHC CR with latest config
shcName := deployment.GetName() + "-shc"
shc := &enterpriseApi.SearchHeadCluster{}
err = deployment.GetInstance(shcName, shc)
Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster")

// Scale Search Head Cluster
defaultSHReplicas := shc.Spec.Replicas
scaledSHReplicas := defaultSHReplicas + 1
testenvInstance.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas)

// Update Replicas of SHC
err = deployment.GetInstance(shcName, shc)
Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster")
shc.Spec.Replicas = int32(scaledSHReplicas)
err = deployment.UpdateCR(shc)
Expect(err).To(Succeed(), "Failed to scale Search Head Cluster")
Expand Down Expand Up @@ -231,6 +244,12 @@ var _ = Describe("c3appfw test", func() {
// Verify RF SF is met
testenv.VerifyRFSFMet(deployment, testenvInstance)

// Verify bundle push status. Bundle hash not compared as scaleup does not involve new config
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledIndexerReplicas), "")
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)
// Saving current V2 bundle hash to future comparision with new config bundle hash
clusterManagerBundleHash = testenv.GetClusterManagerBundleHash(deployment)

// Verify Apps are copied to location
allPodNames = testenv.DumpGetPods(testenvInstance.GetName())
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", "version", appVersion)
Expand Down Expand Up @@ -278,6 +297,12 @@ var _ = Describe("c3appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", " version", appVersion)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation)

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash)
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)
// Saving current V1 bundle hash to future comparision with new config bundle hash
clusterManagerBundleHash = testenv.GetClusterManagerBundleHash(deployment)

// Verify Apps are copied to location
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", " version", appVersion)
testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true)
Expand All @@ -291,7 +316,7 @@ var _ = Describe("c3appfw test", func() {
testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true)

// Delete apps on S3 for new Apps
testenvInstance.Log.Info("Delete Apps on S3 for Version", appVersion)
testenvInstance.Log.Info("Delete Apps on S3 for", " Version", appVersion)
testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps)
uploadedApps = nil

Expand All @@ -315,7 +340,7 @@ var _ = Describe("c3appfw test", func() {
// Wait for the poll period for the apps to be downloaded
time.Sleep(2 * time.Minute)

// Ensure that the cluster-master goes to Ready phase
// Ensure that the cluster-manager goes to Ready phase
testenv.ClusterMasterReady(deployment, testenvInstance)

// Ensure indexers go to Ready phase
Expand All @@ -334,6 +359,10 @@ var _ = Describe("c3appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps version", appVersion)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), masterPodNames, customAppFileList, initContDownloadLocation)

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash)
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)

// Verify Apps are copied to location
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app version", appVersion)
testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, customAppList, true, true)
Expand Down Expand Up @@ -505,9 +534,9 @@ var _ = Describe("c3appfw test", func() {

// Create Single site Cluster and SHC, with App Framework enabled on SHC Deployer
// Deploy the CM
testenvInstance.Log.Info("Deploy Cluster master in single site configuration")
testenvInstance.Log.Info("Deploy Cluster manager in single site configuration")
_, err = deployment.DeployClusterMaster(deployment.GetName(), "", "")
Expect(err).To(Succeed(), "Unable to deploy Cluster Master")
Expect(err).To(Succeed(), "Unable to deploy Cluster Manager")

// Deploy the indexer cluster
testenvInstance.Log.Info("Deploy Indexer Cluster in single site configuration")
Expand Down Expand Up @@ -651,6 +680,19 @@ var _ = Describe("c3appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation)

// Get instance of current SHC CR with latest config
shcName := deployment.GetName() + "-shc"
shc := &enterpriseApi.SearchHeadCluster{}
err = deployment.GetInstance(shcName, shc)
shReplicas := int(shc.Spec.Replicas)
Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster")

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "")
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)
// Saving current V1 bundle hash for future comparision
clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment)

// Verify apps with local scope are installed locally on CM and on SHC Deployer
localPodNames := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}
testenvInstance.Log.Info("Verify Apps are installed Locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion)
Expand All @@ -663,13 +705,6 @@ var _ = Describe("c3appfw test", func() {
clusterPodNames = append(clusterPodNames, string(sh))
}

// Get SH Replicas and add sh pods to pod names
shc := &enterpriseApi.SearchHeadCluster{}
shcName := deployment.GetName() + "-shc"
err = deployment.GetInstance(shcName, shc)
Expect(err).To(Succeed(), "Failed to get instance of SHC")
shReplicas := shc.Spec.Replicas

for i := 0; i < int(shReplicas); i++ {
sh := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i)
clusterPodNames = append(clusterPodNames, string(sh))
Expand Down Expand Up @@ -722,6 +757,12 @@ var _ = Describe("c3appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation)

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash)
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)
// Saving current V2 bundle hash for future comparision
clusterManagerBundleHash = testenv.GetClusterManagerBundleHash(deployment)

// Verify apps with local scope are installed locally on CM and on SHC Deployer
testenvInstance.Log.Info("Verify Apps are installed Locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion)
testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), localPodNames, appListLocal, true, "enabled", true, false)
Expand Down Expand Up @@ -775,6 +816,10 @@ var _ = Describe("c3appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation)

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash)
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)

// Verify apps with local scope are installed locally on CM and on SHC Deployer
testenvInstance.Log.Info("Verify Apps are installed Locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion)
testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), localPodNames, appListLocal, true, "enabled", false, false)
Expand Down Expand Up @@ -842,6 +887,17 @@ var _ = Describe("c3appfw test", func() {
masterPodNames := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), masterPodNames, appFileList, initContDownloadLocation)

// Get instance of current SHC CR with latest config
shcName := deployment.GetName() + "-shc"
shc := &enterpriseApi.SearchHeadCluster{}
err = deployment.GetInstance(shcName, shc)
shReplicas := int(shc.Spec.Replicas)
Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster")

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "")
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)

// Verify apps are copied to location
allPodNames := testenv.DumpGetPods(testenvInstance.GetName())
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND")
Expand All @@ -857,11 +913,7 @@ var _ = Describe("c3appfw test", func() {
sh := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i)
podNames = append(podNames, string(sh))
}
shc := &enterpriseApi.SearchHeadCluster{}
shcName := deployment.GetName() + "-shc"
err = deployment.GetInstance(shcName, shc)
Expect(err).To(Succeed(), "Failed to get instance of SHC")
shReplicas := shc.Spec.Replicas

for i := 0; i < int(shReplicas); i++ {
sh := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i)
podNames = append(podNames, string(sh))
Expand Down
40 changes: 39 additions & 1 deletion test/m4/appframework/appframework_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,19 @@ var _ = Describe("m4appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation)

// Get instance of current SHC CR with latest config
shcName := deployment.GetName() + "-shc"
shc := &enterpriseApi.SearchHeadCluster{}
err = deployment.GetInstance(shcName, shc)
shReplicas := int(shc.Spec.Replicas)
Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster")

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount, "")
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)
// Saving current V1 bundle hash for future comparision
clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment)

// Verify apps are copied to location
allPodNames := testenv.DumpGetPods(testenvInstance.GetName())
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", "version", appVersion)
Expand Down Expand Up @@ -165,6 +178,10 @@ var _ = Describe("m4appfw test", func() {
testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation)

// Verify bundle push status and compare bundle hash with previous V1 bundle
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount, clusterManagerBundleHash)
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)

// Verify apps are copied to location
testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app after upgrade", "version", appVersion)
testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, true)
Expand Down Expand Up @@ -245,6 +262,17 @@ var _ = Describe("m4appfw test", func() {
testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation)

// Get instance of current SHC CR with latest config
shcName := deployment.GetName() + "-shc"
shc := &enterpriseApi.SearchHeadCluster{}
err = deployment.GetInstance(shcName, shc)
shReplicas := int(shc.Spec.Replicas)
Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster")

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount, "")
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)

// Verify apps are copied to location
allPodNames := testenv.DumpGetPods(testenvInstance.GetName())
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app before downgrade", "version", appVersion)
Expand Down Expand Up @@ -282,6 +310,12 @@ var _ = Describe("m4appfw test", func() {
// Verify RF SF is met
testenv.VerifyRFSFMet(deployment, testenvInstance)

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledIndexerReplicas), "")
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount)
// Saving current V2 bundle hash for future comparision
clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment)

// Verify Apps are copied to correct location
allPodNames = testenv.DumpGetPods(testenvInstance.GetName())
testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND after scaling up of indexers", "version", appVersion)
Expand Down Expand Up @@ -311,7 +345,7 @@ var _ = Describe("m4appfw test", func() {
// Wait for the poll period for the apps to be downloaded
time.Sleep(2 * time.Minute)

// Ensure that the cluster-master goes to Ready phase
// Ensure that the cluster-manager goes to Ready phase
testenv.ClusterMasterReady(deployment, testenvInstance)

// Ensure the indexers of all sites go to Ready phase
Expand All @@ -327,6 +361,10 @@ var _ = Describe("m4appfw test", func() {
testenvInstance.Log.Info("Verify older version of apps are downloaded by init container", "version", appVersion)
testenv.VerifyAppsDownloadedByInitContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation)

// Verify bundle push status
testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount, clusterManagerBundleHash)
testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas)

// Verify older version of apps are copied to correct location
testenvInstance.Log.Info("Verify older version of apps are copied to correct location based on Pod KIND", "version", appVersion)
testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true)
Expand Down
Loading