From cad072a7fd6abafd770a1a6a3e8dbaf284bbba00 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 18 Jul 2023 22:44:10 +0000 Subject: [PATCH 1/9] Add integration tests for grafana deployment Signed-off-by: Grant Griffiths --- pkg/util/test/util.go | 171 ++++++++++++++++++ test/integration_test/grafana_test.go | 165 +++++++++++++++++ .../testspec/monitoring/px-versions.yaml | 28 +++ test/integration_test/utils/storagecluster.go | 19 +- 4 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 test/integration_test/grafana_test.go create mode 100644 test/integration_test/testspec/monitoring/px-versions.yaml diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 9ed2ef52b..536c5d736 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -39,6 +39,7 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" certv1 "k8s.io/api/certificates/v1" @@ -163,6 +164,7 @@ var ( opVer1_9_1, _ = version.NewVersion("1.9.1-") opVer1_10, _ = version.NewVersion("1.10.0-") opVer23_3, _ = version.NewVersion("23.3.0-") + opVer23_8, _ = version.NewVersion("23.8.0-") opVer23_5, _ = version.NewVersion("23.5.0-") opVer23_5_1, _ = version.NewVersion("23.5.1-") opVer23_7, _ = version.NewVersion("23.7.0-") @@ -2907,6 +2909,10 @@ func validateStorkSecurityEnvVar(cluster *corev1.StorageCluster, storkDeployment return err } + if err := ValidateGrafana(cluster); err != nil { + return err + } + return nil } @@ -2983,6 +2989,171 @@ func ValidatePrometheus(pxImageList map[string]string, cluster *corev1.StorageCl return nil } +func ValidateGrafana(cluster *corev1.StorageCluster) error { + opVersion, err := GetPxOperatorVersion() + if err != nil { + return err + } + if opVersion.LessThanOrEqual(opVer23_8) { + logrus.Infof("Skipping grafana validation for operation version: [%s]", opVersion.String()) + return nil + } + + shouldBeInstalled := cluster.Spec.Monitoring != nil && cluster.Spec.Monitoring.Grafana != nil && + cluster.Spec.Monitoring.Grafana.Enabled && cluster.Spec.Monitoring.Prometheus != nil && + cluster.Spec.Monitoring.Prometheus.Enabled + err = ValidateGrafanaDeployment(shouldBeInstalled) + if err != nil { + return err + } + err = ValidateGrafanaService(shouldBeInstalled) + if err != nil { + return err + } + + return nil +} + +func ValidateGrafanaImage(shouldBeInstalled bool) error { + err := ValidateGrafanaDeployment(shouldBeInstalled) + if err != nil { + return err + } + + return nil +} + +func ValidateGrafanaDeployment(shouldBeInstalled bool) error { + check := func() (interface{}, bool, error) { + pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"}) + if err != nil { + return nil, true, err + } + if shouldBeInstalled { + if len(pods.Items) > 0 && pods.Items[0].Status.Phase == v1.PodRunning { + logrus.Infof("Grafana installed successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana not yet ready") + return "", true, fmt.Errorf("grafana is not installed when it should be") + } + } else { + if len(pods.Items) < 1 { + logrus.Infof("Grafana uninstalled successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana not yet uninstalled") + return "", true, fmt.Errorf("grafana is installed when it is expected to be uninstalled") + } + } + } + + if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + return err + } + + return nil +} + +func ValidateGrafanaDeploymentImage(image string) error { + check := func() (interface{}, bool, error) { + pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"}) + if err != nil { + return "", true, err + } + if len(pods.Items) > 0 && len(pods.Items[0].Spec.Containers) > 0 { + if pods.Items[0].Spec.Containers[0].Image == image { + logrus.Infof("Grafana installed successfully with image %s", image) + return "", false, nil + } else { + logrus.Errorf("Grafana not yet ready with image %s", image) + return "", true, fmt.Errorf("grafana is not installed with image %s", image) + } + } else { + return "", true, fmt.Errorf("grafana is not installed with image %s", image) + } + } + + if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + return err + } + + return nil +} + +func ValidateGrafanaService(shouldBeInstalled bool) error { + check := func() (interface{}, bool, error) { + svcs, err := coreops.Instance().ListServices("kube-system", metav1.ListOptions{ + LabelSelector: "app=grafana", + }) + if err != nil { + return "", true, err + } + + if shouldBeInstalled { + if len(svcs.Items) > 0 && svcs.Items[0].Spec.Ports[0].Port == 3000 { + logrus.Infof("Grafana svc installed successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana svc not yet ready") + return "", true, fmt.Errorf("grafana is not installed when it should be") + } + } else { + if len(svcs.Items) < 1 { + logrus.Infof("Grafana svc uninstalled successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana svc not yet uninstalled") + return "", true, fmt.Errorf("grafana svc is installed when it is expected to be uninstalled") + } + } + } + + if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + return err + } + + return nil +} + +func ValidateGrafanaConfigmaps(t *testing.T, shouldBeInstalled bool) error { + check := func() (interface{}, bool, error) { + cms, err := coreops.Instance().ListConfigMap("kube-system", metav1.ListOptions{}) + require.NoError(t, err) + + var grafanaConfigmaps []v1.ConfigMap + for _, cm := range cms.Items { + if strings.Contains(cm.Name, "px-grafana-") { + grafanaConfigmaps = append(grafanaConfigmaps, cm) + } + } + + if shouldBeInstalled { + if len(grafanaConfigmaps) == 3 { + logrus.Infof("Grafana configmaps are installed successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana configmaps are not yet ready") + return "", true, fmt.Errorf("grafana is not installed when it should be") + } + } else { + if len(grafanaConfigmaps) < 3 { + logrus.Infof("Grafana configmaps uninstalled successfully") + return "", false, nil + } else { + logrus.Errorf("Grafana configmaps are not yet uninstalled") + return "", true, fmt.Errorf("grafana configmaps are installed when it is expected to be uninstalled") + } + } + } + + if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + return err + } + + return nil +} + // ValidateTelemetryV1Disabled validates telemetry components are uninstalled as expected func ValidateTelemetryV1Disabled(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { t := func() (interface{}, bool, error) { diff --git a/test/integration_test/grafana_test.go b/test/integration_test/grafana_test.go new file mode 100644 index 000000000..07cb036c7 --- /dev/null +++ b/test/integration_test/grafana_test.go @@ -0,0 +1,165 @@ +//go:build integrationtest +// +build integrationtest + +package integrationtest + +import ( + "testing" + "time" + + testutil "github.com/libopenstorage/operator/pkg/util/test" + + corev1 "github.com/libopenstorage/operator/pkg/apis/core/v1" + "github.com/libopenstorage/operator/test/integration_test/types" + ci_utils "github.com/libopenstorage/operator/test/integration_test/utils" + "github.com/portworx/sched-ops/k8s/operator" + "github.com/portworx/sched-ops/task" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + grafanaTestSpecUninstalled = ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "px-test-grafana"}, + Spec: corev1.StorageClusterSpec{ + DeleteStrategy: &corev1.StorageClusterDeleteStrategy{ + Type: corev1.UninstallAndWipeStorageClusterStrategyType, + }, + Monitoring: &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{Enabled: false}, + Grafana: &corev1.GrafanaSpec{Enabled: false}, + Telemetry: &corev1.TelemetrySpec{Enabled: false}, + }, + }, + }) +) + +var grafanaTestCases = []types.TestCase{ + { + TestName: "GrafanaInstall", + TestrailCaseIDs: []string{"XXX", "XXX"}, + TestSpec: grafanaTestSpecUninstalled, + TestFunc: func(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + testSpec := tc.TestSpec(t) + cluster, ok := testSpec.(*corev1.StorageCluster) + require.True(t, ok) + + // Test Case: Grafana disabled by default + logrus.Infof("Validate Grafana not installed by default") + cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) + cluster, err := updateGrafanaInstallation(t, cluster, false, false) + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + + // Test Case: Grafana disabled by default even with prometheus enabled + logrus.Infof("Validate Grafana not installed with only prometheus enabled") + cluster, err = updateGrafanaInstallation(t, cluster, true, false) + require.NoError(t, err) + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + + // Test Case: Grafana not installed when prometheus isn't enabled + logrus.Infof("Validate Grafana not installed when prometheus isn't enabled") + cluster, err = updateGrafanaInstallation(t, cluster, false, true) + require.NoError(t, err) + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + + // Test Case: Grafana installed once enabled with prometheus + logrus.Infof("Validate Grafana installed once prometheus and grafana enabled") + cluster, err = updateGrafanaInstallation(t, cluster, true, true) + require.NoError(t, err) + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + } + }, + }, + { + TestName: "GrafanaInstallVersionManifest", + TestrailCaseIDs: []string{"XXX", "XXX"}, + TestSpec: grafanaTestSpecUninstalled, + TestFunc: func(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + testSpec := tc.TestSpec(t) + cluster, ok := testSpec.(*corev1.StorageCluster) + require.True(t, ok) + err := createVersionsManifest(t) + require.NoError(t, err) + + // Test Case: Grafana disabled by default + logrus.Infof("Validate Grafana not installed by default") + cluster, err = ci_utils.DeployStorageCluster(cluster, ci_utils.PxSpecImages) + require.NoError(t, err) + cluster, err = updateGrafanaInstallation(t, cluster, false, false) + err = testutil.ValidateGrafanaDeploymentImage("docker.io/grafana/grafana:8.5.27") + require.NoError(t, err) + + pxVersionManifest, err := ci_utils.ParseSpecs("monitoring/px-versions.yaml") + require.NoError(t, err) + err = ci_utils.DeleteObjects(pxVersionManifest) + require.NoError(t, err) + } + }, + }, +} + +func createVersionsManifest(t *testing.T) error { + pxVersionManifest, err := ci_utils.ParseSpecs("monitoring/px-versions.yaml") + if err != nil { + return err + } + + err = ci_utils.DeleteObjects(pxVersionManifest) + if err != nil { + return err + } + check := func() (interface{}, bool, error) { + err = ci_utils.CreateObjects(pxVersionManifest) + if err != nil { + return nil, true, err + } + + return nil, false, nil + } + if _, err := task.DoRetryWithTimeout(check, 30*time.Second, 5*time.Second); err != nil { + return err + } + + return nil +} + +func updateGrafanaInstallation(t *testing.T, cluster *corev1.StorageCluster, installPrometheus, installGrafana bool) (*corev1.StorageCluster, error) { + check := func() (interface{}, bool, error) { + cluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace) + if err != nil { + return cluster, true, err + } + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: installPrometheus, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: installGrafana, + }, + } + cluster, err = ci_utils.UpdateStorageCluster(cluster) + if err != nil { + return cluster, true, err + } + + return cluster, false, nil + } + if _, err := task.DoRetryWithTimeout(check, 30*time.Second, 5*time.Second); err != nil { + return cluster, err + } + + return cluster, nil +} + +func TestGrafana(t *testing.T) { + for _, testCase := range grafanaTestCases { + testCase.RunTest(t) + } +} diff --git a/test/integration_test/testspec/monitoring/px-versions.yaml b/test/integration_test/testspec/monitoring/px-versions.yaml new file mode 100644 index 000000000..51b8c35aa --- /dev/null +++ b/test/integration_test/testspec/monitoring/px-versions.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +data: + versions: | + version: 2.13.7 + components: + stork: openstorage/stork:23.5.0 + autopilot: portworx/autopilot:1.3.8 + nodeWiper: portworx/px-node-wiper:2.13.2 + csiNodeDriverRegistrar: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.2 + csiProvisioner: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 + csiAttacher: docker.io/openstorage/csi-attacher:v1.2.1-1 + csiResizer: registry.k8s.io/sig-storage/csi-resizer:v1.6.0 + csiSnapshotter: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + csiSnapshotController: registry.k8s.io/sig-storage/snapshot-controller:v6.1.0 + prometheus: quay.io/prometheus/prometheus:v2.35.0 + prometheusOperator: quay.io/prometheus-operator/prometheus-operator:v0.56.3 + prometheusConfigReloader: quay.io/prometheus-operator/prometheus-config-reloader:v0.56.3 + alertManager: quay.io/prometheus/alertmanager:v0.24.0 + grafana: docker.io/grafana/grafana:8.5.27 + telemetry: purestorage/ccm-go:1.0.3 + pxLibUpdate: portworx/px-lib:pxfslibs-updater + metricsCollector: purestorage/realtime-metrics:1.0.15 + telemetryProxy: purestorage/telemetry-envoy:1.1.6 + logUploader: purestorage/log-upload:px-1.0.12 +kind: ConfigMap +metadata: + name: px-versions + namespace: kube-system diff --git a/test/integration_test/utils/storagecluster.go b/test/integration_test/utils/storagecluster.go index 6bc789517..92cb76e96 100644 --- a/test/integration_test/utils/storagecluster.go +++ b/test/integration_test/utils/storagecluster.go @@ -1,11 +1,12 @@ package utils import ( - "github.com/libopenstorage/cloudops" "path" "strings" "testing" + "github.com/libopenstorage/cloudops" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" @@ -322,6 +323,22 @@ func UpdateAndValidateCSI(cluster *corev1.StorageCluster, f func(*corev1.Storage return latestLiveCluster } +// UpdateAndValidateGrafana update StorageCluster, validates grafana and return latest version of live StorageCluster +func UpdateAndValidateGrafana(cluster *corev1.StorageCluster, f func(*corev1.StorageCluster) *corev1.StorageCluster, pxSpecImages map[string]string, t *testing.T) *corev1.StorageCluster { + liveCluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace) + require.NoError(t, err) + + newCluster := f(liveCluster) + + latestLiveCluster, err := UpdateStorageCluster(newCluster) + require.NoError(t, err) + + err = testutil.ValidateGrafana(cluster) + require.NoError(t, err) + + return latestLiveCluster +} + // UninstallAndValidateStorageCluster uninstall and validate the cluster deletion func UninstallAndValidateStorageCluster(cluster *corev1.StorageCluster, t *testing.T) { // Delete cluster From 1794434064322bb6e083a6134c5472d7784900cf Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 25 Jul 2023 18:14:56 +0000 Subject: [PATCH 2/9] PR Feedback Signed-off-by: Grant Griffiths --- pkg/util/test/util.go | 50 +++++++++++++-------------- test/integration_test/grafana_test.go | 3 +- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 536c5d736..28c859a16 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -2909,10 +2909,6 @@ func validateStorkSecurityEnvVar(cluster *corev1.StorageCluster, storkDeployment return err } - if err := ValidateGrafana(cluster); err != nil { - return err - } - return nil } @@ -2932,6 +2928,9 @@ func ValidateMonitoring(pxImageList map[string]string, cluster *corev1.StorageCl if err := ValidateAlertManager(pxImageList, cluster, timeout, interval); err != nil { return err } + if err := ValidateGrafana(cluster); err != nil { + return err + } return nil } @@ -2994,7 +2993,7 @@ func ValidateGrafana(cluster *corev1.StorageCluster) error { if err != nil { return err } - if opVersion.LessThanOrEqual(opVer23_8) { + if opVersion.LessThan(opVer23_8) { logrus.Infof("Skipping grafana validation for operation version: [%s]", opVersion.String()) return nil } @@ -3002,7 +3001,8 @@ func ValidateGrafana(cluster *corev1.StorageCluster) error { shouldBeInstalled := cluster.Spec.Monitoring != nil && cluster.Spec.Monitoring.Grafana != nil && cluster.Spec.Monitoring.Grafana.Enabled && cluster.Spec.Monitoring.Prometheus != nil && cluster.Spec.Monitoring.Prometheus.Enabled - err = ValidateGrafanaDeployment(shouldBeInstalled) + image := cluster.Status.DesiredImages.Grafana + err = ValidateGrafanaDeployment(shouldBeInstalled, image) if err != nil { return err } @@ -3014,29 +3014,26 @@ func ValidateGrafana(cluster *corev1.StorageCluster) error { return nil } -func ValidateGrafanaImage(shouldBeInstalled bool) error { - err := ValidateGrafanaDeployment(shouldBeInstalled) - if err != nil { - return err - } - - return nil -} - -func ValidateGrafanaDeployment(shouldBeInstalled bool) error { +func ValidateGrafanaDeployment(shouldBeInstalled bool, image string) error { check := func() (interface{}, bool, error) { pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"}) if err != nil { return nil, true, err } if shouldBeInstalled { - if len(pods.Items) > 0 && pods.Items[0].Status.Phase == v1.PodRunning { - logrus.Infof("Grafana installed successfully") - return "", false, nil - } else { + if len(pods.Items) == 0 || pods.Items[0].Status.Phase != v1.PodRunning { logrus.Errorf("Grafana not yet ready") return "", true, fmt.Errorf("grafana is not installed when it should be") } + + err := ValidateGrafanaDeploymentImage(image) + if err != nil { + logrus.Errorf("Grafana image is invalid yet ready") + return "", true, fmt.Errorf("grafana is not installed when it should be") + } + + logrus.Infof("Grafana installed successfully") + return "", false, nil } else { if len(pods.Items) < 1 { logrus.Infof("Grafana uninstalled successfully") @@ -3061,20 +3058,23 @@ func ValidateGrafanaDeploymentImage(image string) error { if err != nil { return "", true, err } - if len(pods.Items) > 0 && len(pods.Items[0].Spec.Containers) > 0 { - if pods.Items[0].Spec.Containers[0].Image == image { + + podCount := len(pods.Items) + if podCount > 0 && len(pods.Items[0].Spec.Containers) > 0 { + actualImage := pods.Items[0].Spec.Containers[0].Image + if actualImage == image { logrus.Infof("Grafana installed successfully with image %s", image) return "", false, nil } else { logrus.Errorf("Grafana not yet ready with image %s", image) - return "", true, fmt.Errorf("grafana is not installed with image %s", image) + return "", true, fmt.Errorf("grafana is not installed with expected image %s. actual: %s", image, actualImage) } } else { - return "", true, fmt.Errorf("grafana is not installed with image %s", image) + return "", true, fmt.Errorf("grafana is not installed with image %s. number of pods: %v", image, podCount) } } - if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + if _, err := task.DoRetryWithTimeout(check, 1*time.Minute, 10*time.Second); err != nil { return err } diff --git a/test/integration_test/grafana_test.go b/test/integration_test/grafana_test.go index 07cb036c7..d2749cc13 100644 --- a/test/integration_test/grafana_test.go +++ b/test/integration_test/grafana_test.go @@ -92,7 +92,8 @@ var grafanaTestCases = []types.TestCase{ logrus.Infof("Validate Grafana not installed by default") cluster, err = ci_utils.DeployStorageCluster(cluster, ci_utils.PxSpecImages) require.NoError(t, err) - cluster, err = updateGrafanaInstallation(t, cluster, false, false) + cluster, err = updateGrafanaInstallation(t, cluster, true, true) + require.NoError(t, err) err = testutil.ValidateGrafanaDeploymentImage("docker.io/grafana/grafana:8.5.27") require.NoError(t, err) From 532b1753041f835047bc83b0d3152b47b2aed59a Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 25 Jul 2023 23:13:07 +0000 Subject: [PATCH 3/9] Refactor test to basic_test.go Signed-off-by: Grant Griffiths --- pkg/util/test/util.go | 22 ++- test/integration_test/basic_test.go | 150 ++++++++++++++++ test/integration_test/grafana_test.go | 166 ------------------ .../testspec/monitoring/px-versions.yaml | 28 --- test/integration_test/utils/storagecluster.go | 2 +- 5 files changed, 164 insertions(+), 204 deletions(-) delete mode 100644 test/integration_test/grafana_test.go delete mode 100644 test/integration_test/testspec/monitoring/px-versions.yaml diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 28c859a16..9815ea07a 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -39,7 +39,6 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" certv1 "k8s.io/api/certificates/v1" @@ -2928,7 +2927,7 @@ func ValidateMonitoring(pxImageList map[string]string, cluster *corev1.StorageCl if err := ValidateAlertManager(pxImageList, cluster, timeout, interval); err != nil { return err } - if err := ValidateGrafana(cluster); err != nil { + if err := ValidateGrafana(pxImageList, cluster); err != nil { return err } @@ -2988,7 +2987,7 @@ func ValidatePrometheus(pxImageList map[string]string, cluster *corev1.StorageCl return nil } -func ValidateGrafana(cluster *corev1.StorageCluster) error { +func ValidateGrafana(pxImageList map[string]string, cluster *corev1.StorageCluster) error { opVersion, err := GetPxOperatorVersion() if err != nil { return err @@ -3001,8 +3000,7 @@ func ValidateGrafana(cluster *corev1.StorageCluster) error { shouldBeInstalled := cluster.Spec.Monitoring != nil && cluster.Spec.Monitoring.Grafana != nil && cluster.Spec.Monitoring.Grafana.Enabled && cluster.Spec.Monitoring.Prometheus != nil && cluster.Spec.Monitoring.Prometheus.Enabled - image := cluster.Status.DesiredImages.Grafana - err = ValidateGrafanaDeployment(shouldBeInstalled, image) + err = ValidateGrafanaDeployment(cluster, shouldBeInstalled, pxImageList) if err != nil { return err } @@ -3010,11 +3008,15 @@ func ValidateGrafana(cluster *corev1.StorageCluster) error { if err != nil { return err } + err = ValidateGrafanaConfigmaps(shouldBeInstalled) + if err != nil { + return err + } return nil } -func ValidateGrafanaDeployment(shouldBeInstalled bool, image string) error { +func ValidateGrafanaDeployment(cluster *corev1.StorageCluster, shouldBeInstalled bool, pxImageList map[string]string) error { check := func() (interface{}, bool, error) { pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"}) if err != nil { @@ -3026,7 +3028,7 @@ func ValidateGrafanaDeployment(shouldBeInstalled bool, image string) error { return "", true, fmt.Errorf("grafana is not installed when it should be") } - err := ValidateGrafanaDeploymentImage(image) + err := validateContainerImageInsidePods(cluster, pxImageList["grafana"], "grafana", pods) if err != nil { logrus.Errorf("Grafana image is invalid yet ready") return "", true, fmt.Errorf("grafana is not installed when it should be") @@ -3116,10 +3118,12 @@ func ValidateGrafanaService(shouldBeInstalled bool) error { return nil } -func ValidateGrafanaConfigmaps(t *testing.T, shouldBeInstalled bool) error { +func ValidateGrafanaConfigmaps(shouldBeInstalled bool) error { check := func() (interface{}, bool, error) { cms, err := coreops.Instance().ListConfigMap("kube-system", metav1.ListOptions{}) - require.NoError(t, err) + if err != nil { + return "", true, err + } var grafanaConfigmaps []v1.ConfigMap for _, cm := range cms.Items { diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index 515bfc3c7..20af4515c 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -170,6 +170,14 @@ var testStorageClusterBasicCases = []types.TestCase{ }), TestFunc: BasicAutopilotRegression, }, + { + TestName: "BasicGrafanaRegression", + TestrailCaseIDs: []string{}, + TestSpec: ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-stc"}, + }), + TestFunc: BasicGrafanaRegression, + }, { TestName: "BasicPvcControllerRegression", TestrailCaseIDs: []string{"C58438", "C54697", "C54698", "C54707", "C58437", "C54476", "C54477"}, @@ -1127,3 +1135,145 @@ func InstallWithNodeTopologyLabels(tc *types.TestCase) func(*testing.T) { } } } + +// BasicGrafanaRegression does the following steps: +// 1. Test Case: Grafana disabled by default +// 2. Test Case: Grafana disabled by default even with prometheus enabled +// 3. Test Case: Grafana not installed when prometheus isn't enabled +// 4. Test Case: Grafana installed once enabled with prometheus +// 5. Test Case: Grafana disabled once prometheus is disabled +// 6. Test Case: Grafana disabled if grafana is disabled as well +// 7. Test Case: Grafana disabled if prometheus spec is removed +// 8. Test Case: Grafana disabled if grafana spec is removed +// 9. Test Case: Grafana disabled if monitoring spec is removed +func BasicGrafanaRegression(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + testSpec := tc.TestSpec(t) + cluster, ok := testSpec.(*corev1.StorageCluster) + require.True(t, ok) + + cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) + + // 1. Test Case: Grafana disabled by default + // Validate monitoring block is nil + require.Nil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should be nil by default, but it seems there is something set in there %+v", cluster.Spec.Monitoring) + + // 2. Test Case: Grafana disabled by default even with prometheus enabled + logrus.Info("Enable prometheus and validate StorageCluster") + updateParamFunc := func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: true, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus) + require.True(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled) + + // 3. Test Case: Grafana not installed when prometheus isn't enabled + logrus.Info("Enable only grafana and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: false, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: true, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana) + require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled) + + // 4. Test Case: Grafana installed once enabled with prometheus + logrus.Info("Enable both grafana and prometheus and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: true, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: true, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana) + require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled) + require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus) + require.True(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled) + + // 5. Test Case: Grafana disabled once prometheus is disabled + logrus.Info("Disable prometheus and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: false, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: true, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana) + require.True(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is enabled: expected: true, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled) + require.NotNil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Prometheus) + require.False(t, cluster.Spec.Monitoring.Prometheus.Enabled, "failed to validate Prometheus is disabled: expected: false, actual: %v", cluster.Spec.Monitoring.Prometheus.Enabled) + + // 6. Test Case: Grafana disabled if grafana is disabled as well + logrus.Info("Disable grafana and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = &corev1.MonitoringSpec{ + Prometheus: &corev1.PrometheusSpec{ + Enabled: false, + }, + Grafana: &corev1.GrafanaSpec{ + Enabled: false, + }, + } + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.NotNil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring) + require.NotNil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should not be nil here, but it is: %+v", cluster.Spec.Monitoring.Grafana) + require.False(t, cluster.Spec.Monitoring.Grafana.Enabled, "failed to validate Grafana is disabled: expected: false, actual: %v", cluster.Spec.Monitoring.Grafana.Enabled) + + // 7. Test Case: Grafana disabled if prometheus spec is removed + logrus.Info("Remove Prometheus block (set it to nil) and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring.Prometheus = nil + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.Nil(t, cluster.Spec.Monitoring.Prometheus, "failed to validate Prometheus block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring.Prometheus) + + // 8. Test Case: Grafana disabled if grafana spec is removed + logrus.Info("Remove Grafana block (set it to nil) and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring.Grafana = nil + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.Nil(t, cluster.Spec.Monitoring.Grafana, "failed to validate Grafana block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring.Grafana) + + // 9. Test Case: Grafana disabled if monitoring spec is removed + logrus.Info("Remove Monitoring block (set it to nil) and validate StorageCluster") + updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster { + cluster.Spec.Monitoring = nil + return cluster + } + cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) + require.Nil(t, cluster.Spec.Monitoring, "failed to validate Monitoring block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring) + } +} diff --git a/test/integration_test/grafana_test.go b/test/integration_test/grafana_test.go deleted file mode 100644 index d2749cc13..000000000 --- a/test/integration_test/grafana_test.go +++ /dev/null @@ -1,166 +0,0 @@ -//go:build integrationtest -// +build integrationtest - -package integrationtest - -import ( - "testing" - "time" - - testutil "github.com/libopenstorage/operator/pkg/util/test" - - corev1 "github.com/libopenstorage/operator/pkg/apis/core/v1" - "github.com/libopenstorage/operator/test/integration_test/types" - ci_utils "github.com/libopenstorage/operator/test/integration_test/utils" - "github.com/portworx/sched-ops/k8s/operator" - "github.com/portworx/sched-ops/task" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var ( - grafanaTestSpecUninstalled = ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: "px-test-grafana"}, - Spec: corev1.StorageClusterSpec{ - DeleteStrategy: &corev1.StorageClusterDeleteStrategy{ - Type: corev1.UninstallAndWipeStorageClusterStrategyType, - }, - Monitoring: &corev1.MonitoringSpec{ - Prometheus: &corev1.PrometheusSpec{Enabled: false}, - Grafana: &corev1.GrafanaSpec{Enabled: false}, - Telemetry: &corev1.TelemetrySpec{Enabled: false}, - }, - }, - }) -) - -var grafanaTestCases = []types.TestCase{ - { - TestName: "GrafanaInstall", - TestrailCaseIDs: []string{"XXX", "XXX"}, - TestSpec: grafanaTestSpecUninstalled, - TestFunc: func(tc *types.TestCase) func(*testing.T) { - return func(t *testing.T) { - testSpec := tc.TestSpec(t) - cluster, ok := testSpec.(*corev1.StorageCluster) - require.True(t, ok) - - // Test Case: Grafana disabled by default - logrus.Infof("Validate Grafana not installed by default") - cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) - cluster, err := updateGrafanaInstallation(t, cluster, false, false) - err = testutil.ValidateGrafana(cluster) - require.NoError(t, err) - - // Test Case: Grafana disabled by default even with prometheus enabled - logrus.Infof("Validate Grafana not installed with only prometheus enabled") - cluster, err = updateGrafanaInstallation(t, cluster, true, false) - require.NoError(t, err) - err = testutil.ValidateGrafana(cluster) - require.NoError(t, err) - - // Test Case: Grafana not installed when prometheus isn't enabled - logrus.Infof("Validate Grafana not installed when prometheus isn't enabled") - cluster, err = updateGrafanaInstallation(t, cluster, false, true) - require.NoError(t, err) - err = testutil.ValidateGrafana(cluster) - require.NoError(t, err) - - // Test Case: Grafana installed once enabled with prometheus - logrus.Infof("Validate Grafana installed once prometheus and grafana enabled") - cluster, err = updateGrafanaInstallation(t, cluster, true, true) - require.NoError(t, err) - err = testutil.ValidateGrafana(cluster) - require.NoError(t, err) - } - }, - }, - { - TestName: "GrafanaInstallVersionManifest", - TestrailCaseIDs: []string{"XXX", "XXX"}, - TestSpec: grafanaTestSpecUninstalled, - TestFunc: func(tc *types.TestCase) func(*testing.T) { - return func(t *testing.T) { - testSpec := tc.TestSpec(t) - cluster, ok := testSpec.(*corev1.StorageCluster) - require.True(t, ok) - err := createVersionsManifest(t) - require.NoError(t, err) - - // Test Case: Grafana disabled by default - logrus.Infof("Validate Grafana not installed by default") - cluster, err = ci_utils.DeployStorageCluster(cluster, ci_utils.PxSpecImages) - require.NoError(t, err) - cluster, err = updateGrafanaInstallation(t, cluster, true, true) - require.NoError(t, err) - err = testutil.ValidateGrafanaDeploymentImage("docker.io/grafana/grafana:8.5.27") - require.NoError(t, err) - - pxVersionManifest, err := ci_utils.ParseSpecs("monitoring/px-versions.yaml") - require.NoError(t, err) - err = ci_utils.DeleteObjects(pxVersionManifest) - require.NoError(t, err) - } - }, - }, -} - -func createVersionsManifest(t *testing.T) error { - pxVersionManifest, err := ci_utils.ParseSpecs("monitoring/px-versions.yaml") - if err != nil { - return err - } - - err = ci_utils.DeleteObjects(pxVersionManifest) - if err != nil { - return err - } - check := func() (interface{}, bool, error) { - err = ci_utils.CreateObjects(pxVersionManifest) - if err != nil { - return nil, true, err - } - - return nil, false, nil - } - if _, err := task.DoRetryWithTimeout(check, 30*time.Second, 5*time.Second); err != nil { - return err - } - - return nil -} - -func updateGrafanaInstallation(t *testing.T, cluster *corev1.StorageCluster, installPrometheus, installGrafana bool) (*corev1.StorageCluster, error) { - check := func() (interface{}, bool, error) { - cluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace) - if err != nil { - return cluster, true, err - } - cluster.Spec.Monitoring = &corev1.MonitoringSpec{ - Prometheus: &corev1.PrometheusSpec{ - Enabled: installPrometheus, - }, - Grafana: &corev1.GrafanaSpec{ - Enabled: installGrafana, - }, - } - cluster, err = ci_utils.UpdateStorageCluster(cluster) - if err != nil { - return cluster, true, err - } - - return cluster, false, nil - } - if _, err := task.DoRetryWithTimeout(check, 30*time.Second, 5*time.Second); err != nil { - return cluster, err - } - - return cluster, nil -} - -func TestGrafana(t *testing.T) { - for _, testCase := range grafanaTestCases { - testCase.RunTest(t) - } -} diff --git a/test/integration_test/testspec/monitoring/px-versions.yaml b/test/integration_test/testspec/monitoring/px-versions.yaml deleted file mode 100644 index 51b8c35aa..000000000 --- a/test/integration_test/testspec/monitoring/px-versions.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: v1 -data: - versions: | - version: 2.13.7 - components: - stork: openstorage/stork:23.5.0 - autopilot: portworx/autopilot:1.3.8 - nodeWiper: portworx/px-node-wiper:2.13.2 - csiNodeDriverRegistrar: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.2 - csiProvisioner: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 - csiAttacher: docker.io/openstorage/csi-attacher:v1.2.1-1 - csiResizer: registry.k8s.io/sig-storage/csi-resizer:v1.6.0 - csiSnapshotter: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 - csiSnapshotController: registry.k8s.io/sig-storage/snapshot-controller:v6.1.0 - prometheus: quay.io/prometheus/prometheus:v2.35.0 - prometheusOperator: quay.io/prometheus-operator/prometheus-operator:v0.56.3 - prometheusConfigReloader: quay.io/prometheus-operator/prometheus-config-reloader:v0.56.3 - alertManager: quay.io/prometheus/alertmanager:v0.24.0 - grafana: docker.io/grafana/grafana:8.5.27 - telemetry: purestorage/ccm-go:1.0.3 - pxLibUpdate: portworx/px-lib:pxfslibs-updater - metricsCollector: purestorage/realtime-metrics:1.0.15 - telemetryProxy: purestorage/telemetry-envoy:1.1.6 - logUploader: purestorage/log-upload:px-1.0.12 -kind: ConfigMap -metadata: - name: px-versions - namespace: kube-system diff --git a/test/integration_test/utils/storagecluster.go b/test/integration_test/utils/storagecluster.go index 92cb76e96..a9a1eef44 100644 --- a/test/integration_test/utils/storagecluster.go +++ b/test/integration_test/utils/storagecluster.go @@ -333,7 +333,7 @@ func UpdateAndValidateGrafana(cluster *corev1.StorageCluster, f func(*corev1.Sto latestLiveCluster, err := UpdateStorageCluster(newCluster) require.NoError(t, err) - err = testutil.ValidateGrafana(cluster) + err = testutil.ValidateGrafana(pxSpecImages, cluster) require.NoError(t, err) return latestLiveCluster From 531c315800d544f45a4ceac8e46379046dda3040 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 26 Jul 2023 18:24:42 +0000 Subject: [PATCH 4/9] PR feedback Signed-off-by: Grant Griffiths --- pkg/util/test/util.go | 154 +++++++++++------------------------------- 1 file changed, 38 insertions(+), 116 deletions(-) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 9815ea07a..2bd1d79fa 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -3004,11 +3004,11 @@ func ValidateGrafana(pxImageList map[string]string, cluster *corev1.StorageClust if err != nil { return err } - err = ValidateGrafanaService(shouldBeInstalled) + err = ValidateGrafanaService(cluster, shouldBeInstalled) if err != nil { return err } - err = ValidateGrafanaConfigmaps(shouldBeInstalled) + err = ValidateGrafanaConfigmaps(cluster, shouldBeInstalled) if err != nil { return err } @@ -3017,145 +3017,67 @@ func ValidateGrafana(pxImageList map[string]string, cluster *corev1.StorageClust } func ValidateGrafanaDeployment(cluster *corev1.StorageCluster, shouldBeInstalled bool, pxImageList map[string]string) error { - check := func() (interface{}, bool, error) { - pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"}) - if err != nil { - return nil, true, err - } - if shouldBeInstalled { - if len(pods.Items) == 0 || pods.Items[0].Status.Phase != v1.PodRunning { - logrus.Errorf("Grafana not yet ready") - return "", true, fmt.Errorf("grafana is not installed when it should be") - } - - err := validateContainerImageInsidePods(cluster, pxImageList["grafana"], "grafana", pods) - if err != nil { - logrus.Errorf("Grafana image is invalid yet ready") - return "", true, fmt.Errorf("grafana is not installed when it should be") - } - - logrus.Infof("Grafana installed successfully") - return "", false, nil - } else { - if len(pods.Items) < 1 { - logrus.Infof("Grafana uninstalled successfully") - return "", false, nil - } else { - logrus.Errorf("Grafana not yet uninstalled") - return "", true, fmt.Errorf("grafana is installed when it is expected to be uninstalled") - } - } + deployment, err := appops.Instance().GetDeployment("px-grafana", cluster.Namespace) + if err != nil { + return err } - if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { + if err := appops.Instance().ValidateDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil { return err } return nil } -func ValidateGrafanaDeploymentImage(image string) error { - check := func() (interface{}, bool, error) { - pods, err := coreops.Instance().GetPods("kube-system", map[string]string{"app": "grafana"}) - if err != nil { - return "", true, err - } - - podCount := len(pods.Items) - if podCount > 0 && len(pods.Items[0].Spec.Containers) > 0 { - actualImage := pods.Items[0].Spec.Containers[0].Image - if actualImage == image { - logrus.Infof("Grafana installed successfully with image %s", image) - return "", false, nil - } else { - logrus.Errorf("Grafana not yet ready with image %s", image) - return "", true, fmt.Errorf("grafana is not installed with expected image %s. actual: %s", image, actualImage) - } - } else { - return "", true, fmt.Errorf("grafana is not installed with image %s. number of pods: %v", image, podCount) - } - } - - if _, err := task.DoRetryWithTimeout(check, 1*time.Minute, 10*time.Second); err != nil { +func ValidateGrafanaService(cluster *corev1.StorageCluster, shouldBeInstalled bool) error { + svcs, err := coreops.Instance().ListServices(cluster.Namespace, metav1.ListOptions{ + LabelSelector: "app=grafana", + }) + if err != nil { return err } - return nil -} - -func ValidateGrafanaService(shouldBeInstalled bool) error { - check := func() (interface{}, bool, error) { - svcs, err := coreops.Instance().ListServices("kube-system", metav1.ListOptions{ - LabelSelector: "app=grafana", - }) - if err != nil { - return "", true, err + if shouldBeInstalled { + if len(svcs.Items) > 0 && svcs.Items[0].Spec.Ports[0].Port == 3000 { + return nil + } else { + return fmt.Errorf("grafana is not installed when it should be") } - - if shouldBeInstalled { - if len(svcs.Items) > 0 && svcs.Items[0].Spec.Ports[0].Port == 3000 { - logrus.Infof("Grafana svc installed successfully") - return "", false, nil - } else { - logrus.Errorf("Grafana svc not yet ready") - return "", true, fmt.Errorf("grafana is not installed when it should be") - } + } else { + if len(svcs.Items) < 1 { + return nil } else { - if len(svcs.Items) < 1 { - logrus.Infof("Grafana svc uninstalled successfully") - return "", false, nil - } else { - logrus.Errorf("Grafana svc not yet uninstalled") - return "", true, fmt.Errorf("grafana svc is installed when it is expected to be uninstalled") - } + return fmt.Errorf("grafana svc is installed when it is expected to be uninstalled") } } +} - if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { +func ValidateGrafanaConfigmaps(cluster *corev1.StorageCluster, shouldBeInstalled bool) error { + cms, err := coreops.Instance().ListConfigMap(cluster.Namespace, metav1.ListOptions{}) + if err != nil { return err } - return nil -} - -func ValidateGrafanaConfigmaps(shouldBeInstalled bool) error { - check := func() (interface{}, bool, error) { - cms, err := coreops.Instance().ListConfigMap("kube-system", metav1.ListOptions{}) - if err != nil { - return "", true, err + var grafanaConfigmaps []v1.ConfigMap + for _, cm := range cms.Items { + if strings.Contains(cm.Name, "px-grafana-") { + grafanaConfigmaps = append(grafanaConfigmaps, cm) } + } - var grafanaConfigmaps []v1.ConfigMap - for _, cm := range cms.Items { - if strings.Contains(cm.Name, "px-grafana-") { - grafanaConfigmaps = append(grafanaConfigmaps, cm) - } + if shouldBeInstalled { + if len(grafanaConfigmaps) == 3 { + return nil + } else { + return fmt.Errorf("grafana is not installed when it should be") } - - if shouldBeInstalled { - if len(grafanaConfigmaps) == 3 { - logrus.Infof("Grafana configmaps are installed successfully") - return "", false, nil - } else { - logrus.Errorf("Grafana configmaps are not yet ready") - return "", true, fmt.Errorf("grafana is not installed when it should be") - } + } else { + if len(grafanaConfigmaps) < 3 { + return nil } else { - if len(grafanaConfigmaps) < 3 { - logrus.Infof("Grafana configmaps uninstalled successfully") - return "", false, nil - } else { - logrus.Errorf("Grafana configmaps are not yet uninstalled") - return "", true, fmt.Errorf("grafana configmaps are installed when it is expected to be uninstalled") - } + return fmt.Errorf("grafana configmaps are installed when it is expected to be uninstalled") } } - - if _, err := task.DoRetryWithTimeout(check, 2*time.Minute, 30*time.Second); err != nil { - return err - } - - return nil } // ValidateTelemetryV1Disabled validates telemetry components are uninstalled as expected From 225bf54474f3397fe31b044af683f487711124b5 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 26 Jul 2023 22:13:03 +0000 Subject: [PATCH 5/9] Handle test conflict cases Signed-off-by: Grant Griffiths --- test/integration_test/basic_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index 20af4515c..857dff947 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -1137,7 +1137,7 @@ func InstallWithNodeTopologyLabels(tc *types.TestCase) func(*testing.T) { } // BasicGrafanaRegression does the following steps: -// 1. Test Case: Grafana disabled by default +// 1. Test Case: Install initial cluster and grafana disabled by default // 2. Test Case: Grafana disabled by default even with prometheus enabled // 3. Test Case: Grafana not installed when prometheus isn't enabled // 4. Test Case: Grafana installed once enabled with prometheus @@ -1152,11 +1152,9 @@ func BasicGrafanaRegression(tc *types.TestCase) func(*testing.T) { cluster, ok := testSpec.(*corev1.StorageCluster) require.True(t, ok) + // 1. Test Case: Install initial cluster and grafana disabled by default cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) - - // 1. Test Case: Grafana disabled by default - // Validate monitoring block is nil - require.Nil(t, cluster.Spec.Monitoring, "failed to validate monitoring block, it should be nil by default, but it seems there is something set in there %+v", cluster.Spec.Monitoring) + require.Nil(t, cluster.Spec.Monitoring.Grafana, "failed to validate grafana block, it should be nil by default, but it seems there is something set in there %+v", cluster.Spec.Monitoring) // 2. Test Case: Grafana disabled by default even with prometheus enabled logrus.Info("Enable prometheus and validate StorageCluster") From 2c4b73fb508ef2799bcc44673a9baad62f22e78c Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Thu, 27 Jul 2023 03:09:31 +0000 Subject: [PATCH 6/9] skip for op 23.7 and earlier Signed-off-by: Grant Griffiths --- test/integration_test/basic_test.go | 3 +++ test/integration_test/utils/px_operator.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index 857dff947..5aef285ee 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -177,6 +177,9 @@ var testStorageClusterBasicCases = []types.TestCase{ ObjectMeta: metav1.ObjectMeta{Name: "test-stc"}, }), TestFunc: BasicGrafanaRegression, + ShouldSkip: func(tc *types.TestCase) bool { + return ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer23_8) + }, }, { TestName: "BasicPvcControllerRegression", diff --git a/test/integration_test/utils/px_operator.go b/test/integration_test/utils/px_operator.go index 9bf43b1e4..29d77576b 100644 --- a/test/integration_test/utils/px_operator.go +++ b/test/integration_test/utils/px_operator.go @@ -28,6 +28,8 @@ var ( PxOperatorVer1_8_1, _ = version.NewVersion("1.8.1-") // PxOperatorVer23_3 portworx-operator 23.3 minimum version PxOperatorVer23_3, _ = version.NewVersion("23.3-") + // PxOperatorVer23_8 portworx-operator 23.8 minimum version + PxOperatorVer23_8, _ = version.NewVersion("23.8-") ) // TODO: Install portworx-operator in test automation From c561fd6d336428ac3085ca82cb4a6aee782e0c20 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Thu, 27 Jul 2023 16:58:13 +0000 Subject: [PATCH 7/9] add skip log line Signed-off-by: Grant Griffiths --- test/integration_test/basic_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index 5aef285ee..d810390cc 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -178,7 +178,11 @@ var testStorageClusterBasicCases = []types.TestCase{ }), TestFunc: BasicGrafanaRegression, ShouldSkip: func(tc *types.TestCase) bool { - return ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer23_8) + skip := ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer23_8) + if skip { + logrus.Info("Skipping BasicGrafanaRegression since operator version is less than 23.8.x") + } + return skip }, }, { From baf4b363ea6058f9e2456774095ce517d15a33d1 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Thu, 27 Jul 2023 17:52:28 +0000 Subject: [PATCH 8/9] Add test debugging Signed-off-by: Grant Griffiths --- pkg/util/test/util.go | 2 +- test/integration_test/basic_test.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 2bd1d79fa..d08af6596 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -3019,7 +3019,7 @@ func ValidateGrafana(pxImageList map[string]string, cluster *corev1.StorageClust func ValidateGrafanaDeployment(cluster *corev1.StorageCluster, shouldBeInstalled bool, pxImageList map[string]string) error { deployment, err := appops.Instance().GetDeployment("px-grafana", cluster.Namespace) if err != nil { - return err + return fmt.Errorf("failed to get deployment in ns %v: %v", cluster.Namespace, err) } if err := appops.Instance().ValidateDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil { diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index d810390cc..5ce59a09a 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -1280,5 +1280,8 @@ func BasicGrafanaRegression(tc *types.TestCase) func(*testing.T) { } cluster = ci_utils.UpdateAndValidateGrafana(cluster, updateParamFunc, ci_utils.PxSpecImages, t) require.Nil(t, cluster.Spec.Monitoring, "failed to validate Monitoring block, it should be nil here, but it is not: %+v", cluster.Spec.Monitoring) + + // Delete and validate StorageCluster deletion + ci_utils.UninstallAndValidateStorageCluster(cluster, t) } } From 7e6da52fe23576358ea82245b8872573d8f06c2e Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Thu, 27 Jul 2023 22:17:40 +0000 Subject: [PATCH 9/9] Check for grafana running properly Signed-off-by: Grant Griffiths --- pkg/util/test/util.go | 29 +++++++++++++------ test/integration_test/utils/storagecluster.go | 2 +- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index d08af6596..b96b39ca7 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -2997,9 +2997,9 @@ func ValidateGrafana(pxImageList map[string]string, cluster *corev1.StorageClust return nil } - shouldBeInstalled := cluster.Spec.Monitoring != nil && cluster.Spec.Monitoring.Grafana != nil && - cluster.Spec.Monitoring.Grafana.Enabled && cluster.Spec.Monitoring.Prometheus != nil && - cluster.Spec.Monitoring.Prometheus.Enabled + shouldBeInstalled := cluster.Spec.Monitoring != nil && + cluster.Spec.Monitoring.Grafana != nil && cluster.Spec.Monitoring.Grafana.Enabled && + cluster.Spec.Monitoring.Prometheus != nil && cluster.Spec.Monitoring.Prometheus.Enabled err = ValidateGrafanaDeployment(cluster, shouldBeInstalled, pxImageList) if err != nil { return err @@ -3017,13 +3017,24 @@ func ValidateGrafana(pxImageList map[string]string, cluster *corev1.StorageClust } func ValidateGrafanaDeployment(cluster *corev1.StorageCluster, shouldBeInstalled bool, pxImageList map[string]string) error { - deployment, err := appops.Instance().GetDeployment("px-grafana", cluster.Namespace) - if err != nil { - return fmt.Errorf("failed to get deployment in ns %v: %v", cluster.Namespace, err) - } - if err := appops.Instance().ValidateDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil { - return err + // Deployment to validate + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-grafana", + Namespace: cluster.Namespace, + }, + } + if shouldBeInstalled { + if err := appops.Instance().ValidateDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil { + return fmt.Errorf("failed to validate deployment %s/%s. should be installed: %v. err: %v", + deployment.Namespace, deployment.Name, shouldBeInstalled, err) + } + } else { + if err := appops.Instance().ValidateTerminatedDeployment(deployment, 2*time.Minute, 10*time.Second); err != nil { + return fmt.Errorf("failed to validate terminated deployment %s/%s. should be installed: %v. err: %v", + deployment.Namespace, deployment.Name, shouldBeInstalled, err) + } } return nil diff --git a/test/integration_test/utils/storagecluster.go b/test/integration_test/utils/storagecluster.go index a9a1eef44..0a6e65fe8 100644 --- a/test/integration_test/utils/storagecluster.go +++ b/test/integration_test/utils/storagecluster.go @@ -333,7 +333,7 @@ func UpdateAndValidateGrafana(cluster *corev1.StorageCluster, f func(*corev1.Sto latestLiveCluster, err := UpdateStorageCluster(newCluster) require.NoError(t, err) - err = testutil.ValidateGrafana(pxSpecImages, cluster) + err = testutil.ValidateGrafana(pxSpecImages, latestLiveCluster) require.NoError(t, err) return latestLiveCluster