Add integration tests for grafana deployment

Signed-off-by: Grant Griffiths <ggriffiths@purestorage.com>

pkg/util/test/util.go

@@ -24,6 +24,7 @@ import (
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
 	appsv1 "k8s.io/api/apps/v1"
 	certv1 "k8s.io/api/certificates/v1"
@@ -140,6 +141,7 @@ var (
 	opVer1_9_1, _                     = version.NewVersion("1.9.1-")
 	opVer1_10, _                      = version.NewVersion("1.10.0-")
 	opVer23_3, _                      = version.NewVersion("23.3.0-")
+	opVer23_8, _                      = version.NewVersion("23.8.0-")
 	opVer23_5, _                      = version.NewVersion("23.5.0-")
 	minOpVersionForKubeSchedConfig, _ = version.NewVersion("1.10.2-")
 )
@@ -2832,6 +2834,10 @@ func ValidateMonitoring(pxImageList map[string]string, cluster *corev1.StorageCl
 		return err
 	}
 
+	if err := ValidateGrafana(cluster); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -2888,6 +2894,171 @@ func ValidatePrometheus(pxImageList map[string]string, cluster *corev1.StorageCl
 	return nil
 }
 
+func ValidateGrafana(cluster *corev1.StorageCluster) error {
+	opVersion, err := GetPxOperatorVersion()
+	if err != nil {
+		return err
+	}
+	if opVersion.LessThanOrEqual(opVer23_8) {
+		logrus.Infof("Skipping grafana validation for operation version: [%s]", opVersion.String())
+		return nil
+	}
+
+	shouldBeInstalled := cluster.Spec.Monitoring != nil && cluster.Spec.Monitoring.Grafana != nil &&
+		cluster.Spec.Monitoring.Grafana.Enabled && cluster.Spec.Monitoring.Prometheus != nil &&
+		cluster.Spec.Monitoring.Prometheus.Enabled
+	err = ValidateGrafanaDeployment(shouldBeInstalled)
+	if err != nil {
+		return err
+	}
+	err = ValidateGrafanaService(shouldBeInstalled)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ValidateGrafanaImage(shouldBeInstalled bool) error {
+	err := ValidateGrafanaDeployment(shouldBeInstalled)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ValidateGrafanaDeployment(shouldBeInstalled bool) error {
+	check := func() (interface{}, bool, error) {
+		pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"})
+		if err != nil {
+			return nil, true, err
+		}
+		if shouldBeInstalled {
+			if len(pods.Items) > 0 && pods.Items[0].Status.Phase == v1.PodRunning {
+				logrus.Infof("Grafana installed successfully")
+				return "", false, nil
+			} else {
+				logrus.Errorf("Grafana not yet ready")
+				return "", true, fmt.Errorf("grafana is not installed when it should be")
+			}
+		} else {
+			if len(pods.Items) < 1 {
+				logrus.Infof("Grafana uninstalled successfully")
+				return "", false, nil
+			} else {
+				logrus.Errorf("Grafana not yet uninstalled")
+				return "", true, fmt.Errorf("grafana is installed when it is expected to be uninstalled")
+			}
+		}
+	}
+
+	if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ValidateGrafanaDeploymentImage(image string) error {
+	check := func() (interface{}, bool, error) {
+		pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"})
+		if err != nil {
+			return "", true, err
+		}
+		if len(pods.Items) > 0 && len(pods.Items[0].Spec.Containers) > 0 {
+			if pods.Items[0].Spec.Containers[0].Image == image {
+				logrus.Infof("Grafana installed successfully with image %s", image)
+				return "", false, nil
+			} else {
+				logrus.Errorf("Grafana not yet ready with image %s", image)
+				return "", true, fmt.Errorf("grafana is not installed with image %s", image)
+			}
+		} else {
+			return "", true, fmt.Errorf("grafana is not installed with image %s", image)
+		}
+	}
+
+	if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ValidateGrafanaService(shouldBeInstalled bool) error {
+	check := func() (interface{}, bool, error) {
+		svcs, err := coreops.Instance().ListServices("kube-system", metav1.ListOptions{
+			LabelSelector: "app=grafana",
+		})
+		if err != nil {
+			return "", true, err
+		}
+
+		if shouldBeInstalled {
+			if len(svcs.Items) > 0 && svcs.Items[0].Spec.Ports[0].Port == 3000 {
+				logrus.Infof("Grafana svc installed successfully")
+				return "", false, nil
+			} else {
+				logrus.Errorf("Grafana svc not yet ready")
+				return "", true, fmt.Errorf("grafana is not installed when it should be")
+			}
+		} else {
+			if len(svcs.Items) < 1 {
+				logrus.Infof("Grafana svc uninstalled successfully")
+				return "", false, nil
+			} else {
+				logrus.Errorf("Grafana svc not yet uninstalled")
+				return "", true, fmt.Errorf("grafana svc is installed when it is expected to be uninstalled")
+			}
+		}
+	}
+
+	if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ValidateGrafanaConfigmaps(t *testing.T, shouldBeInstalled bool) error {
+	check := func() (interface{}, bool, error) {
+		cms, err := coreops.Instance().ListConfigMap("kube-system", metav1.ListOptions{})
+		require.NoError(t, err)
+
+		var grafanaConfigmaps []v1.ConfigMap
+		for _, cm := range cms.Items {
+			if strings.Contains(cm.Name, "px-grafana-") {
+				grafanaConfigmaps = append(grafanaConfigmaps, cm)
+			}
+		}
+
+		if shouldBeInstalled {
+			if len(grafanaConfigmaps) == 3 {
+				logrus.Infof("Grafana configmaps are installed successfully")
+				return "", false, nil
+			} else {
+				logrus.Errorf("Grafana configmaps are not yet ready")
+				return "", true, fmt.Errorf("grafana is not installed when it should be")
+			}
+		} else {
+			if len(grafanaConfigmaps) < 3 {
+				logrus.Infof("Grafana configmaps uninstalled successfully")
+				return "", false, nil
+			} else {
+				logrus.Errorf("Grafana configmaps are not yet uninstalled")
+				return "", true, fmt.Errorf("grafana configmaps are installed when it is expected to be uninstalled")
+			}
+		}
+	}
+
+	if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // ValidateTelemetryV1Disabled validates telemetry components are uninstalled as expected
 func ValidateTelemetryV1Disabled(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
 	t := func() (interface{}, bool, error) {

test/integration_test/grafana_test.go

@@ -0,0 +1,165 @@
+//go:build integrationtest
+// +build integrationtest
+
+package integrationtest
+
+import (
+	"testing"
+	"time"
+
+	testutil "github.com/libopenstorage/operator/pkg/util/test"
+
+	corev1 "github.com/libopenstorage/operator/pkg/apis/core/v1"
+	"github.com/libopenstorage/operator/test/integration_test/types"
+	ci_utils "github.com/libopenstorage/operator/test/integration_test/utils"
+	"github.com/portworx/sched-ops/k8s/operator"
+	"github.com/portworx/sched-ops/task"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	grafanaTestSpecUninstalled = ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "px-test-grafana"},
+		Spec: corev1.StorageClusterSpec{
+			DeleteStrategy: &corev1.StorageClusterDeleteStrategy{
+				Type: corev1.UninstallAndWipeStorageClusterStrategyType,
+			},
+			Monitoring: &corev1.MonitoringSpec{
+				Prometheus: &corev1.PrometheusSpec{Enabled: false},
+				Grafana:    &corev1.GrafanaSpec{Enabled: false},
+				Telemetry:  &corev1.TelemetrySpec{Enabled: false},
+			},
+		},
+	})
+)
+
+var grafanaTestCases = []types.TestCase{
+	{
+		TestName:        "GrafanaInstall",
+		TestrailCaseIDs: []string{"XXX", "XXX"},
+		TestSpec:        grafanaTestSpecUninstalled,
+		TestFunc: func(tc *types.TestCase) func(*testing.T) {
+			return func(t *testing.T) {
+				testSpec := tc.TestSpec(t)
+				cluster, ok := testSpec.(*corev1.StorageCluster)
+				require.True(t, ok)
+
+				// Test Case: Grafana disabled by default
+				logrus.Infof("Validate Grafana not installed by default")
+				cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t)
+				cluster, err := updateGrafanaInstallation(t, cluster, false, false)
+				err = testutil.ValidateGrafana(cluster)
+				require.NoError(t, err)
+
+				// Test Case: Grafana disabled by default even with prometheus enabled
+				logrus.Infof("Validate Grafana not installed with only prometheus enabled")
+				cluster, err = updateGrafanaInstallation(t, cluster, true, false)
+				require.NoError(t, err)
+				err = testutil.ValidateGrafana(cluster)
+				require.NoError(t, err)
+
+				// Test Case: Grafana not installed when prometheus isn't enabled
+				logrus.Infof("Validate Grafana not installed when prometheus isn't enabled")
+				cluster, err = updateGrafanaInstallation(t, cluster, false, true)
+				require.NoError(t, err)
+				err = testutil.ValidateGrafana(cluster)
+				require.NoError(t, err)
+
+				// Test Case: Grafana installed once enabled with prometheus
+				logrus.Infof("Validate Grafana installed once prometheus and grafana enabled")
+				cluster, err = updateGrafanaInstallation(t, cluster, true, true)
+				require.NoError(t, err)
+				err = testutil.ValidateGrafana(cluster)
+				require.NoError(t, err)
+			}
+		},
+	},
+	{
+		TestName:        "GrafanaInstallVersionManifest",
+		TestrailCaseIDs: []string{"XXX", "XXX"},
+		TestSpec:        grafanaTestSpecUninstalled,
+		TestFunc: func(tc *types.TestCase) func(*testing.T) {
+			return func(t *testing.T) {
+				testSpec := tc.TestSpec(t)
+				cluster, ok := testSpec.(*corev1.StorageCluster)
+				require.True(t, ok)
+				err := createVersionsManifest(t)
+				require.NoError(t, err)
+
+				// Test Case: Grafana disabled by default
+				logrus.Infof("Validate Grafana not installed by default")
+				cluster, err = ci_utils.DeployStorageCluster(cluster, ci_utils.PxSpecImages)
+				require.NoError(t, err)
+				cluster, err = updateGrafanaInstallation(t, cluster, false, false)
+				err = testutil.ValidateGrafanaDeploymentImage("docker.io/grafana/grafana:8.5.27")
+				require.NoError(t, err)
+
+				pxVersionManifest, err := ci_utils.ParseSpecs("monitoring/px-versions.yaml")
+				require.NoError(t, err)
+				err = ci_utils.DeleteObjects(pxVersionManifest)
+				require.NoError(t, err)
+			}
+		},
+	},
+}
+
+func createVersionsManifest(t *testing.T) error {
+	pxVersionManifest, err := ci_utils.ParseSpecs("monitoring/px-versions.yaml")
+	if err != nil {
+		return err
+	}
+
+	err = ci_utils.DeleteObjects(pxVersionManifest)
+	if err != nil {
+		return err
+	}
+	check := func() (interface{}, bool, error) {
+		err = ci_utils.CreateObjects(pxVersionManifest)
+		if err != nil {
+			return nil, true, err
+		}
+
+		return nil, false, nil
+	}
+	if _, err := task.DoRetryWithTimeout(check, 30*time.Second, 5*time.Second); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func updateGrafanaInstallation(t *testing.T, cluster *corev1.StorageCluster, installPrometheus, installGrafana bool) (*corev1.StorageCluster, error) {
+	check := func() (interface{}, bool, error) {
+		cluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace)
+		if err != nil {
+			return cluster, true, err
+		}
+		cluster.Spec.Monitoring = &corev1.MonitoringSpec{
+			Prometheus: &corev1.PrometheusSpec{
+				Enabled: installPrometheus,
+			},
+			Grafana: &corev1.GrafanaSpec{
+				Enabled: installGrafana,
+			},
+		}
+		cluster, err = ci_utils.UpdateStorageCluster(cluster)
+		if err != nil {
+			return cluster, true, err
+		}
+
+		return cluster, false, nil
+	}
+	if _, err := task.DoRetryWithTimeout(check, 30*time.Second, 5*time.Second); err != nil {
+		return cluster, err
+	}
+
+	return cluster, nil
+}
+
+func TestGrafana(t *testing.T) {
+	for _, testCase := range grafanaTestCases {
+		testCase.RunTest(t)
+	}
+}

test/integration_test/testspec/monitoring/px-versions.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+data:
+  versions: |
+    version: 2.13.7
+    components:
+      stork: openstorage/stork:23.5.0
+      autopilot: portworx/autopilot:1.3.8
+      nodeWiper: portworx/px-node-wiper:2.13.2
+      csiNodeDriverRegistrar: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.2
+      csiProvisioner: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
+      csiAttacher: docker.io/openstorage/csi-attacher:v1.2.1-1
+      csiResizer: registry.k8s.io/sig-storage/csi-resizer:v1.6.0
+      csiSnapshotter: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0
+      csiSnapshotController: registry.k8s.io/sig-storage/snapshot-controller:v6.1.0
+      prometheus: quay.io/prometheus/prometheus:v2.35.0
+      prometheusOperator: quay.io/prometheus-operator/prometheus-operator:v0.56.3
+      prometheusConfigReloader: quay.io/prometheus-operator/prometheus-config-reloader:v0.56.3
+      alertManager: quay.io/prometheus/alertmanager:v0.24.0
+      grafana: docker.io/grafana/grafana:8.5.27
+      telemetry: purestorage/ccm-go:1.0.3
+      pxLibUpdate: portworx/px-lib:pxfslibs-updater
+      metricsCollector: purestorage/realtime-metrics:1.0.15
+      telemetryProxy: purestorage/telemetry-envoy:1.1.6
+      logUploader: purestorage/log-upload:px-1.0.12
+kind: ConfigMap
+metadata:
+  name: px-versions
+  namespace: kube-system