Add integration tests for grafana deployment (#1156)

* Add integration tests for grafana deployment Signed-off-by: Grant Griffiths <ggriffiths@purestorage.com>
libopenstorage · Jul 28, 2023 · 8078b73 · 8078b73
1 parent 591d5b5
commit 8078b73
Show file tree

Hide file tree

Showing 4 changed files with 286 additions and 1 deletion.
diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go
@@ -163,6 +163,7 @@ var (
 	opVer1_9_1, _                     = version.NewVersion("1.9.1-")
 	opVer1_10, _                      = version.NewVersion("1.10.0-")
 	opVer23_3, _                      = version.NewVersion("23.3.0-")
+	opVer23_8, _                      = version.NewVersion("23.8.0-")
 	opVer23_5, _                      = version.NewVersion("23.5.0-")
 	opVer23_5_1, _                    = version.NewVersion("23.5.1-")
 	opVer23_7, _                      = version.NewVersion("23.7.0-")
@@ -2926,6 +2927,9 @@ func ValidateMonitoring(pxImageList map[string]string, cluster *corev1.StorageCl
 	if err := ValidateAlertManager(pxImageList, cluster, timeout, interval); err != nil {
 		return err
 	}
+	if err := ValidateGrafana(pxImageList, cluster); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -2983,6 +2987,110 @@ func ValidatePrometheus(pxImageList map[string]string, cluster *corev1.StorageCl
 	return nil
 }
 
+func ValidateGrafana(pxImageList map[string]string, cluster *corev1.StorageCluster) error {
+	opVersion, err := GetPxOperatorVersion()
+	if err != nil {
+		return err
+	}
+	if opVersion.LessThan(opVer23_8) {
+		logrus.Infof("Skipping grafana validation for operation version: [%s]", opVersion.String())
+		return nil
+	}
+
+	shouldBeInstalled := cluster.Spec.Monitoring != nil &&
+		cluster.Spec.Monitoring.Grafana != nil && cluster.Spec.Monitoring.Grafana.Enabled &&
+		cluster.Spec.Monitoring.Prometheus != nil && cluster.Spec.Monitoring.Prometheus.Enabled
+	err = ValidateGrafanaDeployment(cluster, shouldBeInstalled, pxImageList)
+	if err != nil {
+		return err
+	}
+	err = ValidateGrafanaService(cluster, shouldBeInstalled)
+	if err != nil {
+		return err
+	}
+	err = ValidateGrafanaConfigmaps(cluster, shouldBeInstalled)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ValidateGrafanaDeployment(cluster *corev1.StorageCluster, shouldBeInstalled bool, pxImageList map[string]string) error {
+
+	// Deployment to validate
+	deployment := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "px-grafana",
+			Namespace: cluster.Namespace,
+		},
+	}
+	if shouldBeInstalled {
+		if err := appops.Instance().ValidateDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil {
+			return fmt.Errorf("failed to validate deployment %s/%s. should be installed: %v. err: %v",
+				deployment.Namespace, deployment.Name, shouldBeInstalled, err)
+		}
+	} else {
+		if err := appops.Instance().ValidateTerminatedDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil {
+			return fmt.Errorf("failed to validate terminated deployment %s/%s. should be installed: %v. err: %v",
+				deployment.Namespace, deployment.Name, shouldBeInstalled, err)
+		}
+	}
+
+	return nil
+}
+
+func ValidateGrafanaService(cluster *corev1.StorageCluster, shouldBeInstalled bool) error {
+	svcs, err := coreops.Instance().ListServices(cluster.Namespace, metav1.ListOptions{
+		LabelSelector: "app=grafana",
+	})
+	if err != nil {
+		return err
+	}
+
+	if shouldBeInstalled {
+		if len(svcs.Items) > 0 && svcs.Items[0].Spec.Ports[0].Port == 3000 {
+			return nil
+		} else {
+			return fmt.Errorf("grafana is not installed when it should be")
+		}
+	} else {
+		if len(svcs.Items) < 1 {
+			return nil
+		} else {
+			return fmt.Errorf("grafana svc is installed when it is expected to be uninstalled")
+		}
+	}
+}
+
+func ValidateGrafanaConfigmaps(cluster *corev1.StorageCluster, shouldBeInstalled bool) error {
+	cms, err := coreops.Instance().ListConfigMap(cluster.Namespace, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	var grafanaConfigmaps []v1.ConfigMap
+	for _, cm := range cms.Items {
+		if strings.Contains(cm.Name, "px-grafana-") {
+			grafanaConfigmaps = append(grafanaConfigmaps, cm)
+		}
+	}
+
+	if shouldBeInstalled {
+		if len(grafanaConfigmaps) == 3 {
+			return nil
+		} else {
+			return fmt.Errorf("grafana is not installed when it should be")
+		}
+	} else {
+		if len(grafanaConfigmaps) < 3 {
+			return nil
+		} else {
+			return fmt.Errorf("grafana configmaps are installed when it is expected to be uninstalled")
+		}
+	}
+}
+
 // ValidateTelemetryV1Disabled validates telemetry components are uninstalled as expected
 func ValidateTelemetryV1Disabled(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
 	t := func() (interface{}, bool, error) {

diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go
@@ -170,6 +170,21 @@ var testStorageClusterBasicCases = []types.TestCase{
 		}),
 		TestFunc: BasicAutopilotRegression,
 	},
+	{
+		TestName:        "BasicGrafanaRegression",
+		TestrailCaseIDs: []string{},
+		TestSpec: ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{
+			ObjectMeta: metav1.ObjectMeta{Name: "test-stc"},
+		}),
+		TestFunc: BasicGrafanaRegression,
+		ShouldSkip: func(tc *types.TestCase) bool {
+			skip := ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer23_8)
+			if skip {
+				logrus.Info("Skipping BasicGrafanaRegression since operator version is less than 23.8.x")
+			}
+			return skip
+		},
+	},
 	{
 		TestName:        "BasicPvcControllerRegression",
 		TestrailCaseIDs: []string{"C58438", "C54697", "C54698", "C54707", "C58437", "C54476", "C54477"},
@@ -1127,3 +1142,146 @@ func InstallWithNodeTopologyLabels(tc *types.TestCase) func(*testing.T) {
 		}
 	}
 }
+
+// BasicGrafanaRegression does the following steps:
+// 1. Test Case: Install initial cluster and grafana disabled by default
+// 2. Test Case: Grafana disabled by default even with prometheus enabled
+// 3. Test Case: Grafana not installed when prometheus isn't enabled
+// 4. Test Case: Grafana installed once enabled with prometheus
+// 5. Test Case: Grafana disabled once prometheus is disabled
+// 6. Test Case: Grafana disabled if grafana is disabled as well
+// 7. Test Case: Grafana disabled if prometheus spec is removed
+// 8. Test Case: Grafana disabled if grafana spec is removed
+// 9. Test Case: Grafana disabled if monitoring spec is removed
+func BasicGrafanaRegression(tc *types.TestCase) func(*testing.T) {
+	return func(t *testing.T) {
+		testSpec := tc.TestSpec(t)
+		cluster, ok := testSpec.(*corev1.StorageCluster)
+		require.True(t, ok)
+
+		// 1. Test Case: Install initial cluster and grafana disabled by default
+		cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t)
+		require.Nil(t, cluster.Spec.Monitoring.Grafana, "failed to validate grafana block, it should be nil by default, but it seems there is something set in there %+v", cluster.Spec.Monitoring)
+
+		// 2. Test Case: Grafana disabled by default even with prometheus enabled
+		logrus.Info("Enable prometheus and validate StorageCluster")
+		updateParamFunc := func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
+			cluster.Spec.Monitoring = &corev1.MonitoringSpec{
+				Prometheus: &corev1.PrometheusSpec{
+					Enabled: true,
+				},
+			}
+			return cluster
+		}
+		cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
+		require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring)
+		require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus)
+		require.True(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled)
+
+		// 3. Test Case: Grafana not installed when prometheus isn't enabled
+		logrus.Info("Enable only grafana and validate StorageCluster")
+		updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
+			cluster.Spec.Monitoring = &corev1.MonitoringSpec{
+				Prometheus: &corev1.PrometheusSpec{
+					Enabled: false,
+				},
+				Grafana: &corev1.GrafanaSpec{
+					Enabled: true,
+				},
+			}
+			return cluster
+		}
+		cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
+		require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring)
+		require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana)
+		require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled)
+
+		// 4. Test Case: Grafana installed once enabled with prometheus
+		logrus.Info("Enable both grafana and prometheus and validate StorageCluster")
+		updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
+			cluster.Spec.Monitoring = &corev1.MonitoringSpec{
+				Prometheus: &corev1.PrometheusSpec{
+					Enabled: true,
+				},
+				Grafana: &corev1.GrafanaSpec{
+					Enabled: true,
+				},
+			}
+			return cluster
+		}
+		cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
+		require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring)
+		require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana)
+		require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled)
+		require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus)
+		require.True(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled)
+
+		// 5. Test Case: Grafana disabled once prometheus is disabled
+		logrus.Info("Disable prometheus and validate StorageCluster")
+		updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
+			cluster.Spec.Monitoring = &corev1.MonitoringSpec{
+				Prometheus: &corev1.PrometheusSpec{
+					Enabled: false,
+				},
+				Grafana: &corev1.GrafanaSpec{
+					Enabled: true,
+				},
+			}
+			return cluster
+		}
+		cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
+		require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring)
+		require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana)
+		require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled)
+		require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus)
+		require.False(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is disabled: expected: false, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled)
+
+		// 6. Test Case: Grafana disabled if grafana is disabled as well
+		logrus.Info("Disable grafana and validate StorageCluster")
+		updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
+			cluster.Spec.Monitoring = &corev1.MonitoringSpec{
+				Prometheus: &corev1.PrometheusSpec{
+					Enabled: false,
+				},
+				Grafana: &corev1.GrafanaSpec{
+					Enabled: false,
+				},
+			}
+			return cluster
+		}
+		cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
+		require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring)
+		require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana)
+		require.False(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is disabled: expected: false, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled)
+
+		// 7. Test Case: Grafana disabled if prometheus spec is removed
+		logrus.Info("Remove Prometheus block (set it to nil) and validate StorageCluster")
+		updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
+			cluster.Spec.Monitoring.Prometheus = nil
+			return cluster
+		}
+		cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
+		require.Nil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring.Prometheus)
+
+		// 8. Test Case: Grafana disabled if grafana spec is removed
+		logrus.Info("Remove Grafana block (set it to nil) and validate StorageCluster")
+		updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
+			cluster.Spec.Monitoring.Grafana = nil
+			return cluster
+		}
+		cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
+		require.Nil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring.Grafana)
+
+		// 9. Test Case: Grafana disabled if monitoring spec is removed
+		logrus.Info("Remove Monitoring block (set it to nil) and validate StorageCluster")
+		updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
+			cluster.Spec.Monitoring = nil
+			return cluster
+		}
+		cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
+		require.Nil(t, cluster.Spec.Monitoring, "failed to validate Monitoring block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring)
+
+		// Delete and validate StorageCluster deletion
+		ci_utils.UninstallAndValidateStorageCluster(cluster, t)
+	}
+}
diff --git a/test/integration_test/utils/px_operator.go b/test/integration_test/utils/px_operator.go
@@ -28,6 +28,8 @@ var (
 	PxOperatorVer1_8_1, _ = version.NewVersion("1.8.1-")
 	// PxOperatorVer23_3 portworx-operator 23.3 minimum version
 	PxOperatorVer23_3, _ = version.NewVersion("23.3-")
+	// PxOperatorVer23_8 portworx-operator 23.8 minimum version
+	PxOperatorVer23_8, _ = version.NewVersion("23.8-")
 )
 
 // TODO: Install portworx-operator in test automation

diff --git a/test/integration_test/utils/storagecluster.go b/test/integration_test/utils/storagecluster.go
@@ -1,11 +1,12 @@
 package utils
 
 import (
-	"github.com/libopenstorage/cloudops"
 	"path"
 	"strings"
 	"testing"
 
+	"github.com/libopenstorage/cloudops"
+
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
@@ -322,6 +323,22 @@ func UpdateAndValidateCSI(cluster *corev1.StorageCluster, f func(*corev1.Storage
 	return latestLiveCluster
 }
 
+// UpdateAndValidateGrafana update StorageCluster, validates grafana and return latest version of live StorageCluster
+func UpdateAndValidateGrafana(cluster *corev1.StorageCluster, f func(*corev1.StorageCluster) *corev1.StorageCluster, pxSpecImages map[string]string, t *testing.T) *corev1.StorageCluster {
+	liveCluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace)
+	require.NoError(t, err)
+
+	newCluster := f(liveCluster)
+
+	latestLiveCluster, err := UpdateStorageCluster(newCluster)
+	require.NoError(t, err)
+
+	err = testutil.ValidateGrafana(pxSpecImages, latestLiveCluster)
+	require.NoError(t, err)
+
+	return latestLiveCluster
+}
+
 // UninstallAndValidateStorageCluster uninstall and validate the cluster deletion
 func UninstallAndValidateStorageCluster(cluster *corev1.StorageCluster, t *testing.T) {
 	// Delete cluster