Enable beats stack monitoring configuration

test/e2e/test/beat/builder.go

@@ -350,10 +350,25 @@ func (b Builder) GetMetricsIndexPattern() string {
 	return ".monitoring-beat-*"
 }
 
+func (b Builder) Name() string {
+	return b.Beat.Name
+}
+
+func (b Builder) Namespace() string {
+	return b.Beat.Namespace
+}
+
 func (b Builder) GetMetricsCluster() *types.NamespacedName {
 	if len(b.Beat.Spec.Monitoring.ElasticsearchRefs) == 0 {
 		return nil
 	}
 	metricsCluster := b.Beat.Spec.Monitoring.ElasticsearchRefs[0].NamespacedName()
 	return &metricsCluster
 }
+
+// GetLogsCluster does not return a logs cluster, as Beats stack monitoring is slightly
+// different that both Elasticsearch, and Kibana stack monitoring, as it uses internal
+// beat collectors to send only metrics data, not logs data.
+func (b Builder) GetLogsCluster() *types.NamespacedName {
+	return nil
+}

test/e2e/test/beat/checks.go

@@ -10,11 +10,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"strconv"
 
 	beatcommon "github.com/elastic/cloud-on-k8s/v2/pkg/controller/beat/common"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client"
-	"github.com/elastic/cloud-on-k8s/v2/test/e2e/test/checks"
 )
 
 func HasEventFromBeat(name beatcommon.Type) ValidationFunc {
@@ -89,38 +87,3 @@ func checkEvent(url string, check func(int) error) ValidationFunc {
 		return check(len(results.Hits.Hits))
 	}
 }
-
-func containsDocuments(esClient client.Client, indexPattern string) error {
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/_cat/indices/%s?format=json", indexPattern), nil) //nolint:noctx
-	if err != nil {
-		return err
-	}
-	resp, err := esClient.Request(context.Background(), req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	resultBytes, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	var indices []checks.Index
-	err = json.Unmarshal(resultBytes, &indices)
-	if err != nil {
-		return err
-	}
-
-	// 1 index must exist
-	if len(indices) != 1 {
-		return fmt.Errorf("expected [%d] index [%s], found [%d]", 1, indexPattern, len(indices))
-	}
-	docsCount, err := strconv.Atoi(indices[0].DocsCount)
-	if err != nil {
-		return err
-	}
-	// with at least 1 doc
-	if docsCount <= 0 {
-		return fmt.Errorf("index [%s] empty", indexPattern)
-	}
-	return nil
-}

test/e2e/test/beat/steps.go

@@ -19,6 +19,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/v2/pkg/utils/pointer"
 	"github.com/elastic/cloud-on-k8s/v2/test/e2e/cmd/run"
 	"github.com/elastic/cloud-on-k8s/v2/test/e2e/test"
+	"github.com/elastic/cloud-on-k8s/v2/test/e2e/test/checks"
 	"github.com/elastic/cloud-on-k8s/v2/test/e2e/test/elasticsearch"
 	"github.com/elastic/cloud-on-k8s/v2/test/e2e/test/generation"
 )
@@ -184,7 +185,7 @@ func (b Builder) CheckStackTestSteps(k *test.K8sClient) test.StepList {
 				return nil
 			}),
 		},
-		b.CheckMonitoringMetricsIndex(k),
+		checks.BeatsMonitoredStep(&b, k),
 	}
 }
 
@@ -253,31 +254,3 @@ func (b Builder) MutationTestSteps(k *test.K8sClient) test.StepList {
 		WithSteps(b.CheckStackTestSteps(k)).
 		WithStep(generation.CompareObjectGenerationsStep(&b.Beat, k, isMutated, beatGenerationBeforeMutation, beatObservedGenerationBeforeMutation))
 }
-
-func (b Builder) CheckMonitoringMetricsIndex(k *test.K8sClient) test.Step {
-	indexPattern := b.GetMetricsIndexPattern()
-	return test.Step{
-		Name: fmt.Sprintf("Check that documents are indexed in index %s", indexPattern),
-		Test: test.Eventually(func() error {
-			if b.GetMetricsCluster() == nil {
-				return nil
-			}
-			esMetricsRef := *b.GetMetricsCluster()
-			// Get Elasticsearch
-			esMetrics := esv1.Elasticsearch{}
-			if err := k.Client.Get(context.Background(), esMetricsRef, &esMetrics); err != nil {
-				return err
-			}
-			// Create a new Elasticsearch client
-			client, err := elasticsearch.NewElasticsearchClient(esMetrics, k)
-			if err != nil {
-				return err
-			}
-			// Check that there is at least one document
-			err = containsDocuments(client, indexPattern)
-			if err != nil {
-				return err
-			}
-			return nil
-		})}
-}

test/e2e/test/checks/monitoring.go

@@ -36,6 +36,13 @@ func MonitoredSteps(monitored Monitored, k8sClient *test.K8sClient) test.StepLis
 	}.Steps()
 }
 
+func BeatsMonitoredStep(monitored Monitored, k8sClient *test.K8sClient) test.Step {
+	return stackMonitoringChecks{
+		monitored: monitored,
+		k8sClient: k8sClient,
+	}.BeatsStep()
+}
+
 // stackMonitoringChecks tests that the monitored resource pods have 3 containers ready and that there are documents indexed in the beat indexes
 // of the monitoring Elasticsearch clusters.
 type stackMonitoringChecks struct {
@@ -51,6 +58,10 @@ func (c stackMonitoringChecks) Steps() test.StepList {
 	}
 }
 
+func (c stackMonitoringChecks) BeatsStep() test.Step {
+	return c.CheckMonitoringMetricsIndex()
+}
+
 func (c stackMonitoringChecks) CheckBeatSidecars() test.Step {
 	return test.Step{
 		Name: "Check that beat sidecars are running",