From 8078b739434cf4c13890fcf47253cbf48eebacda Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Thu, 27 Jul 2023 21:01:54 -0700 Subject: [PATCH] Add integration tests for grafana deployment (#1156) * Add integration tests for grafana deployment Signed-off-by: Grant Griffiths --- pkg/util/test/util.go | 108 ++++++++++++ test/integration_test/basic_test.go | 158 ++++++++++++++++++ test/integration_test/utils/px_operator.go | 2 + test/integration_test/utils/storagecluster.go | 19 ++- 4 files changed, 286 insertions(+), 1 deletion(-) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 9ed2ef52b..b96b39ca7 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -163,6 +163,7 @@ var ( opVer1_9_1, _ = version.NewVersion("1.9.1-") opVer1_10, _ = version.NewVersion("1.10.0-") opVer23_3, _ = version.NewVersion("23.3.0-") + opVer23_8, _ = version.NewVersion("23.8.0-") opVer23_5, _ = version.NewVersion("23.5.0-") opVer23_5_1, _ = version.NewVersion("23.5.1-") opVer23_7, _ = version.NewVersion("23.7.0-") @@ -2926,6 +2927,9 @@ func ValidateMonitoring(pxImageList map[string]string, cluster *corev1.StorageCl if err := ValidateAlertManager(pxImageList, cluster, timeout, interval); err != nil { return err } + if err := ValidateGrafana(pxImageList, cluster); err != nil { + return err + } return nil } @@ -2983,6 +2987,110 @@ func ValidatePrometheus(pxImageList map[string]string, cluster *corev1.StorageCl return nil } +func ValidateGrafana(pxImageList map[string]string, cluster *corev1.StorageCluster) error { + opVersion, err := GetPxOperatorVersion() + if err != nil { + return err + } + if opVersion.LessThan(opVer23_8) { + logrus.Infof("Skipping grafana validation for operation version: [%s]", opVersion.String()) + return nil + } + + shouldBeInstalled := cluster.Spec.Monitoring != nil && + cluster.Spec.Monitoring.Grafana != nil && cluster.Spec.Monitoring.Grafana.Enabled && + cluster.Spec.Monitoring.Prometheus != nil && cluster.Spec.Monitoring.Prometheus.Enabled + err = ValidateGrafanaDeployment(cluster, shouldBeInstalled, pxImageList) + if err != nil { + return err + } + err = ValidateGrafanaService(cluster, shouldBeInstalled) + if err != nil { + return err + } + err = ValidateGrafanaConfigmaps(cluster, shouldBeInstalled) + if err != nil { + return err + } + + return nil +} + +func ValidateGrafanaDeployment(cluster *corev1.StorageCluster, shouldBeInstalled bool, pxImageList map[string]string) error { + + // Deployment to validate + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-grafana", + Namespace: cluster.Namespace, + }, + } + if shouldBeInstalled { + if err := appops.Instance().ValidateDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil { + return fmt.Errorf("failed to validate deployment %s/%s. should be installed: %v. err: %v", + deployment.Namespace, deployment.Name, shouldBeInstalled, err) + } + } else { + if err := appops.Instance().ValidateTerminatedDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil { + return fmt.Errorf("failed to validate terminated deployment %s/%s. should be installed: %v. err: %v", + deployment.Namespace, deployment.Name, shouldBeInstalled, err) + } + } + + return nil +} + +func ValidateGrafanaService(cluster *corev1.StorageCluster, shouldBeInstalled bool) error { + svcs, err := coreops.Instance().ListServices(cluster.Namespace, metav1.ListOptions{ + LabelSelector: "app=grafana", + }) + if err != nil { + return err + } + + if shouldBeInstalled { + if len(svcs.Items) > 0 && svcs.Items[0].Spec.Ports[0].Port == 3000 { + return nil + } else { + return fmt.Errorf("grafana is not installed when it should be") + } + } else { + if len(svcs.Items) < 1 { + return nil + } else { + return fmt.Errorf("grafana svc is installed when it is expected to be uninstalled") + } + } +} + +func ValidateGrafanaConfigmaps(cluster *corev1.StorageCluster, shouldBeInstalled bool) error { + cms, err := coreops.Instance().ListConfigMap(cluster.Namespace, metav1.ListOptions{}) + if err != nil { + return err + } + + var grafanaConfigmaps []v1.ConfigMap + for _, cm := range cms.Items { + if strings.Contains(cm.Name, "px-grafana-") { + grafanaConfigmaps = append(grafanaConfigmaps, cm) + } + } + + if shouldBeInstalled { + if len(grafanaConfigmaps) == 3 { + return nil + } else { + return fmt.Errorf("grafana is not installed when it should be") + } + } else { + if len(grafanaConfigmaps) < 3 { + return nil + } else { + return fmt.Errorf("grafana configmaps are installed when it is expected to be uninstalled") + } + } +} + // ValidateTelemetryV1Disabled validates telemetry components are uninstalled as expected func ValidateTelemetryV1Disabled(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { t := func() (interface{}, bool, error) { diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index 515bfc3c7..5ce59a09a 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -170,6 +170,21 @@ var testStorageClusterBasicCases = []types.TestCase{ }), TestFunc: BasicAutopilotRegression, }, + { + TestName: "BasicGrafanaRegression", + TestrailCaseIDs: []string{}, + TestSpec: ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-stc"}, + }), + TestFunc: BasicGrafanaRegression, + ShouldSkip: func(tc *types.TestCase) bool { + skip := ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer23_8) + if skip { + logrus.Info("Skipping BasicGrafanaRegression since operator version is less than 23.8.x") + } + return skip + }, + }, { TestName: "BasicPvcControllerRegression", TestrailCaseIDs: []string{"C58438", "C54697", "C54698", "C54707", "C58437", "C54476", "C54477"}, @@ -1127,3 +1142,146 @@ func InstallWithNodeTopologyLabels(tc *types.TestCase) func(*testing.T) { } } } + +// BasicGrafanaRegression does the following steps: +// 1. Test Case: Install initial cluster and grafana disabled by default +// 2. Test Case: Grafana disabled by default even with prometheus enabled +// 3. Test Case: Grafana not installed when prometheus isn't enabled +// 4. Test Case: Grafana installed once enabled with prometheus +// 5. Test Case: Grafana disabled once prometheus is disabled +// 6. Test Case: Grafana disabled if grafana is disabled as well +// 7. Test Case: Grafana disabled if prometheus spec is removed +// 8. Test Case: Grafana disabled if grafana spec is removed +// 9. Test Case: Grafana disabled if monitoring spec is removed +func BasicGrafanaRegression(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + testSpec := tc.TestSpec(t) + cluster, ok := testSpec.(*corev1.StorageCluster) + require.True(t, ok) + + // 1. Test Case: Install initial cluster and grafana disabled by default + cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) + require.Nil(t, cluster.Spec.Monitoring.Grafana, "failed to validate grafana block, it should be nil by default, but it seems there is something set in there %+v", cluster.Spec.Monitoring) + + // 2. Test Case: Grafana disabled by default even with prometheus enabled + logrus.Info("Enable prometheus and validate StorageCluster") + updateParamFunc := func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: true, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus) + require.True(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled) + + // 3. Test Case: Grafana not installed when prometheus isn't enabled + logrus.Info("Enable only grafana and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: false, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: true, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana) + require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled) + + // 4. Test Case: Grafana installed once enabled with prometheus + logrus.Info("Enable both grafana and prometheus and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: true, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: true, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana) + require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled) + require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus) + require.True(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled) + + // 5. Test Case: Grafana disabled once prometheus is disabled + logrus.Info("Disable prometheus and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: false, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: true, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana) + require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled) + require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus) + require.False(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is disabled: expected: false, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled) + + // 6. Test Case: Grafana disabled if grafana is disabled as well + logrus.Info("Disable grafana and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: false, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: false, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana) + require.False(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is disabled: expected: false, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled) + + // 7. Test Case: Grafana disabled if prometheus spec is removed + logrus.Info("Remove Prometheus block (set it to nil) and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring.Prometheus = nil + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.Nil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring.Prometheus) + + // 8. Test Case: Grafana disabled if grafana spec is removed + logrus.Info("Remove Grafana block (set it to nil) and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring.Grafana = nil + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.Nil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring.Grafana) + + // 9. Test Case: Grafana disabled if monitoring spec is removed + logrus.Info("Remove Monitoring block (set it to nil) and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = nil + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.Nil(t, cluster.Spec.Monitoring, "failed to validate Monitoring block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring) + + // Delete and validate StorageCluster deletion + ci_utils.UninstallAndValidateStorageCluster(cluster, t) + } +} diff --git a/test/integration_test/utils/px_operator.go b/test/integration_test/utils/px_operator.go index 9bf43b1e4..29d77576b 100644 --- a/test/integration_test/utils/px_operator.go +++ b/test/integration_test/utils/px_operator.go @@ -28,6 +28,8 @@ var ( PxOperatorVer1_8_1, _ = version.NewVersion("1.8.1-") // PxOperatorVer23_3 portworx-operator 23.3 minimum version PxOperatorVer23_3, _ = version.NewVersion("23.3-") + // PxOperatorVer23_8 portworx-operator 23.8 minimum version + PxOperatorVer23_8, _ = version.NewVersion("23.8-") ) // TODO: Install portworx-operator in test automation diff --git a/test/integration_test/utils/storagecluster.go b/test/integration_test/utils/storagecluster.go index 6bc789517..0a6e65fe8 100644 --- a/test/integration_test/utils/storagecluster.go +++ b/test/integration_test/utils/storagecluster.go @@ -1,11 +1,12 @@ package utils import ( - "github.com/libopenstorage/cloudops" "path" "strings" "testing" + "github.com/libopenstorage/cloudops" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" @@ -322,6 +323,22 @@ func UpdateAndValidateCSI(cluster *corev1.StorageCluster, f func(*corev1.Storage return latestLiveCluster } +// UpdateAndValidateGrafana update StorageCluster, validates grafana and return latest version of live StorageCluster +func UpdateAndValidateGrafana(cluster *corev1.StorageCluster, f func(*corev1.StorageCluster) *corev1.StorageCluster, pxSpecImages map[string]string, t *testing.T) *corev1.StorageCluster { + liveCluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace) + require.NoError(t, err) + + newCluster := f(liveCluster) + + latestLiveCluster, err := UpdateStorageCluster(newCluster) + require.NoError(t, err) + + err = testutil.ValidateGrafana(pxSpecImages, latestLiveCluster) + require.NoError(t, err) + + return latestLiveCluster +} + // UninstallAndValidateStorageCluster uninstall and validate the cluster deletion func UninstallAndValidateStorageCluster(cluster *corev1.StorageCluster, t *testing.T) { // Delete cluster