diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index f7d3b6fb39..a0fed0d8e7 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -24,6 +24,7 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" certv1 "k8s.io/api/certificates/v1" @@ -140,6 +141,7 @@ var ( opVer1_9_1, _ = version.NewVersion("1.9.1-") opVer1_10, _ = version.NewVersion("1.10.0-") opVer23_3, _ = version.NewVersion("23.3.0-") + opVer23_8, _ = version.NewVersion("23.8.0-") opVer23_5, _ = version.NewVersion("23.5.0-") minOpVersionForKubeSchedConfig, _ = version.NewVersion("1.10.2-") ) @@ -2832,6 +2834,10 @@ func ValidateMonitoring(pxImageList map[string]string, cluster *corev1.StorageCl return err } + if err := ValidateGrafana(cluster); err != nil { + return err + } + return nil } @@ -2888,6 +2894,171 @@ func ValidatePrometheus(pxImageList map[string]string, cluster *corev1.StorageCl return nil } +func ValidateGrafana(cluster *corev1.StorageCluster) error { + opVersion, err := GetPxOperatorVersion() + if err != nil { + return err + } + if opVersion.LessThanOrEqual(opVer23_8) { + logrus.Infof("Skipping grafana validation for operation version: [%s]", opVersion.String()) + return nil + } + + shouldBeInstalled := cluster.Spec.Monitoring != nil && cluster.Spec.Monitoring.Grafana != nil && + cluster.Spec.Monitoring.Grafana.Enabled && cluster.Spec.Monitoring.Prometheus != nil && + cluster.Spec.Monitoring.Prometheus.Enabled + err = ValidateGrafanaDeployment(shouldBeInstalled) + if err != nil { + return err + } + err = ValidateGrafanaService(shouldBeInstalled) + if err != nil { + return err + } + + return nil +} + +func ValidateGrafanaImage(shouldBeInstalled bool) error { + err := ValidateGrafanaDeployment(shouldBeInstalled) + if err != nil { + return err + } + + return nil +} + +func ValidateGrafanaDeployment(shouldBeInstalled bool) error { + check := func() (interface{}, bool, error) { + pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"}) + if err != nil { + return nil, true, err + } + if shouldBeInstalled { + if len(pods.Items) > 0 && pods.Items[0].Status.Phase == v1.PodRunning { + logrus.Infof("Grafana installed successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana not yet ready") + return "", true, fmt.Errorf("grafana is not installed when it should be") + } + } else { + if len(pods.Items) < 1 { + logrus.Infof("Grafana uninstalled successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana not yet uninstalled") + return "", true, fmt.Errorf("grafana is installed when it is expected to be uninstalled") + } + } + } + + if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + return err + } + + return nil +} + +func ValidateGrafanaDeploymentImage(image string) error { + check := func() (interface{}, bool, error) { + pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"}) + if err != nil { + return "", true, err + } + if len(pods.Items) > 0 && len(pods.Items[0].Spec.Containers) > 0 { + if pods.Items[0].Spec.Containers[0].Image == image { + logrus.Infof("Grafana installed successfully with image %s", image) + return "", false, nil + } else { + logrus.Errorf("Grafana not yet ready with image %s", image) + return "", true, fmt.Errorf("grafana is not installed with image %s", image) + } + } else { + return "", true, fmt.Errorf("grafana is not installed with image %s", image) + } + } + + if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + return err + } + + return nil +} + +func ValidateGrafanaService(shouldBeInstalled bool) error { + check := func() (interface{}, bool, error) { + svcs, err := coreops.Instance().ListServices("kube-system", metav1.ListOptions{ + LabelSelector: "app=grafana", + }) + if err != nil { + return "", true, err + } + + if shouldBeInstalled { + if len(svcs.Items) > 0 && svcs.Items[0].Spec.Ports[0].Port == 3000 { + logrus.Infof("Grafana svc installed successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana svc not yet ready") + return "", true, fmt.Errorf("grafana is not installed when it should be") + } + } else { + if len(svcs.Items) < 1 { + logrus.Infof("Grafana svc uninstalled successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana svc not yet uninstalled") + return "", true, fmt.Errorf("grafana svc is installed when it is expected to be uninstalled") + } + } + } + + if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + return err + } + + return nil +} + +func ValidateGrafanaConfigmaps(t *testing.T, shouldBeInstalled bool) error { + check := func() (interface{}, bool, error) { + cms, err := coreops.Instance().ListConfigMap("kube-system", metav1.ListOptions{}) + require.NoError(t, err) + + var grafanaConfigmaps []v1.ConfigMap + for _, cm := range cms.Items { + if strings.Contains(cm.Name, "px-grafana-") { + grafanaConfigmaps = append(grafanaConfigmaps, cm) + } + } + + if shouldBeInstalled { + if len(grafanaConfigmaps) == 3 { + logrus.Infof("Grafana configmaps are installed successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana configmaps are not yet ready") + return "", true, fmt.Errorf("grafana is not installed when it should be") + } + } else { + if len(grafanaConfigmaps) < 3 { + logrus.Infof("Grafana configmaps uninstalled successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana configmaps are not yet uninstalled") + return "", true, fmt.Errorf("grafana configmaps are installed when it is expected to be uninstalled") + } + } + } + + if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + return err + } + + return nil +} + // ValidateTelemetryV1Disabled validates telemetry components are uninstalled as expected func ValidateTelemetryV1Disabled(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { t := func() (interface{}, bool, error) { diff --git a/test/integration_test/grafana_test.go b/test/integration_test/grafana_test.go new file mode 100644 index 0000000000..07cb036c79 --- /dev/null +++ b/test/integration_test/grafana_test.go @@ -0,0 +1,165 @@ +//go:build integrationtest +// +build integrationtest + +package integrationtest + +import ( + "testing" + "time" + + testutil "github.com/libopenstorage/operator/pkg/util/test" + + corev1 "github.com/libopenstorage/operator/pkg/apis/core/v1" + "github.com/libopenstorage/operator/test/integration_test/types" + ci_utils "github.com/libopenstorage/operator/test/integration_test/utils" + "github.com/portworx/sched-ops/k8s/operator" + "github.com/portworx/sched-ops/task" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + grafanaTestSpecUninstalled = ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "px-test-grafana"}, + Spec: corev1.StorageClusterSpec{ + DeleteStrategy: &corev1.StorageClusterDeleteStrategy{ + Type: corev1.UninstallAndWipeStorageClusterStrategyType, + }, + Monitoring: &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{Enabled: false}, + Grafana: &corev1.GrafanaSpec{Enabled: false}, + Telemetry: &corev1.TelemetrySpec{Enabled: false}, + }, + }, + }) +) + +var grafanaTestCases = []types.TestCase{ + { + TestName: "GrafanaInstall", + TestrailCaseIDs: []string{"XXX", "XXX"}, + TestSpec: grafanaTestSpecUninstalled, + TestFunc: func(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + testSpec := tc.TestSpec(t) + cluster, ok := testSpec.(*corev1.StorageCluster) + require.True(t, ok) + + // Test Case: Grafana disabled by default + logrus.Infof("Validate Grafana not installed by default") + cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) + cluster, err := updateGrafanaInstallation(t, cluster, false, false) + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + + // Test Case: Grafana disabled by default even with prometheus enabled + logrus.Infof("Validate Grafana not installed with only prometheus enabled") + cluster, err = updateGrafanaInstallation(t, cluster, true, false) + require.NoError(t, err) + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + + // Test Case: Grafana not installed when prometheus isn't enabled + logrus.Infof("Validate Grafana not installed when prometheus isn't enabled") + cluster, err = updateGrafanaInstallation(t, cluster, false, true) + require.NoError(t, err) + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + + // Test Case: Grafana installed once enabled with prometheus + logrus.Infof("Validate Grafana installed once prometheus and grafana enabled") + cluster, err = updateGrafanaInstallation(t, cluster, true, true) + require.NoError(t, err) + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + } + }, + }, + { + TestName: "GrafanaInstallVersionManifest", + TestrailCaseIDs: []string{"XXX", "XXX"}, + TestSpec: grafanaTestSpecUninstalled, + TestFunc: func(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + testSpec := tc.TestSpec(t) + cluster, ok := testSpec.(*corev1.StorageCluster) + require.True(t, ok) + err := createVersionsManifest(t) + require.NoError(t, err) + + // Test Case: Grafana disabled by default + logrus.Infof("Validate Grafana not installed by default") + cluster, err = ci_utils.DeployStorageCluster(cluster, ci_utils.PxSpecImages) + require.NoError(t, err) + cluster, err = updateGrafanaInstallation(t, cluster, false, false) + err = testutil.ValidateGrafanaDeploymentImage("docker.io/grafana/grafana:8.5.27") + require.NoError(t, err) + + pxVersionManifest, err := ci_utils.ParseSpecs("monitoring/px-versions.yaml") + require.NoError(t, err) + err = ci_utils.DeleteObjects(pxVersionManifest) + require.NoError(t, err) + } + }, + }, +} + +func createVersionsManifest(t *testing.T) error { + pxVersionManifest, err := ci_utils.ParseSpecs("monitoring/px-versions.yaml") + if err != nil { + return err + } + + err = ci_utils.DeleteObjects(pxVersionManifest) + if err != nil { + return err + } + check := func() (interface{}, bool, error) { + err = ci_utils.CreateObjects(pxVersionManifest) + if err != nil { + return nil, true, err + } + + return nil, false, nil + } + if _, err := task.DoRetryWithTimeout(check, 30*time.Second, 5*time.Second); err != nil { + return err + } + + return nil +} + +func updateGrafanaInstallation(t *testing.T, cluster *corev1.StorageCluster, installPrometheus, installGrafana bool) (*corev1.StorageCluster, error) { + check := func() (interface{}, bool, error) { + cluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace) + if err != nil { + return cluster, true, err + } + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: installPrometheus, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: installGrafana, + }, + } + cluster, err = ci_utils.UpdateStorageCluster(cluster) + if err != nil { + return cluster, true, err + } + + return cluster, false, nil + } + if _, err := task.DoRetryWithTimeout(check, 30*time.Second, 5*time.Second); err != nil { + return cluster, err + } + + return cluster, nil +} + +func TestGrafana(t *testing.T) { + for _, testCase := range grafanaTestCases { + testCase.RunTest(t) + } +} diff --git a/test/integration_test/testspec/monitoring/px-versions.yaml b/test/integration_test/testspec/monitoring/px-versions.yaml new file mode 100644 index 0000000000..51b8c35aae --- /dev/null +++ b/test/integration_test/testspec/monitoring/px-versions.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +data: + versions: | + version: 2.13.7 + components: + stork: openstorage/stork:23.5.0 + autopilot: portworx/autopilot:1.3.8 + nodeWiper: portworx/px-node-wiper:2.13.2 + csiNodeDriverRegistrar: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.2 + csiProvisioner: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 + csiAttacher: docker.io/openstorage/csi-attacher:v1.2.1-1 + csiResizer: registry.k8s.io/sig-storage/csi-resizer:v1.6.0 + csiSnapshotter: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + csiSnapshotController: registry.k8s.io/sig-storage/snapshot-controller:v6.1.0 + prometheus: quay.io/prometheus/prometheus:v2.35.0 + prometheusOperator: quay.io/prometheus-operator/prometheus-operator:v0.56.3 + prometheusConfigReloader: quay.io/prometheus-operator/prometheus-config-reloader:v0.56.3 + alertManager: quay.io/prometheus/alertmanager:v0.24.0 + grafana: docker.io/grafana/grafana:8.5.27 + telemetry: purestorage/ccm-go:1.0.3 + pxLibUpdate: portworx/px-lib:pxfslibs-updater + metricsCollector: purestorage/realtime-metrics:1.0.15 + telemetryProxy: purestorage/telemetry-envoy:1.1.6 + logUploader: purestorage/log-upload:px-1.0.12 +kind: ConfigMap +metadata: + name: px-versions + namespace: kube-system diff --git a/test/integration_test/utils/storagecluster.go b/test/integration_test/utils/storagecluster.go index f9a1cbb198..0583e0e4b3 100644 --- a/test/integration_test/utils/storagecluster.go +++ b/test/integration_test/utils/storagecluster.go @@ -1,11 +1,12 @@ package utils import ( - "github.com/libopenstorage/cloudops" "path" "strings" "testing" + "github.com/libopenstorage/cloudops" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" @@ -322,6 +323,22 @@ func UpdateAndValidateCSI(cluster *corev1.StorageCluster, f func(*corev1.Storage return latestLiveCluster } +// UpdateAndValidateGrafana update StorageCluster, validates grafana and return latest version of live StorageCluster +func UpdateAndValidateGrafana(cluster *corev1.StorageCluster, f func(*corev1.StorageCluster) *corev1.StorageCluster, pxSpecImages map[string]string, t *testing.T) *corev1.StorageCluster { + liveCluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace) + require.NoError(t, err) + + newCluster := f(liveCluster) + + latestLiveCluster, err := UpdateStorageCluster(newCluster) + require.NoError(t, err) + + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + + return latestLiveCluster +} + // UninstallAndValidateStorageCluster uninstall and validate the cluster deletion func UninstallAndValidateStorageCluster(cluster *corev1.StorageCluster, t *testing.T) { // Delete cluster