Enable beats stack monitoring configuration

cmd/manager/main.go

@@ -817,6 +817,7 @@ func registerControllers(mgr manager.Manager, params operator.Parameters, access
 		{name: "EMS-ES", registerFunc: associationctl.AddMapsES},
 		{name: "ES-MONITORING", registerFunc: associationctl.AddEsMonitoring},
 		{name: "KB-MONITORING", registerFunc: associationctl.AddKbMonitoring},
+		{name: "BEAT-MONITORING", registerFunc: associationctl.AddBeatMonitoring},
 	}
 
 	for _, c := range assocControllers {

config/crds/v1/all-crds.yaml

@@ -2351,6 +2351,54 @@ spec:
                       is used.
                     type: string
                 type: object
+              monitoring:
+                description: Monitoring enables you to collect and ship monitoring
+                  data of this Beat. See https://www.elastic.co/guide/en/beats/filebeat/current/monitoring.html
+                  Internal Beat collectors are configured and send metrics data to
+                  one Elasticsearch monitoring cluster running in the same Kubernetes
+                  cluster.
+                properties:
+                  elasticsearchRefs:
+                    description: ElasticsearchRefs is a reference to a list of monitoring
+                      Elasticsearch clusters running in the same Kubernetes cluster.
+                      Due to existing limitations, only a single Elasticsearch cluster
+                      is currently supported.
+                    items:
+                      description: ObjectSelector defines a reference to a Kubernetes
+                        object which can be an Elastic resource managed by the operator
+                        or a Secret describing an external Elastic resource not managed
+                        by the operator.
+                      properties:
+                        name:
+                          description: Name of an existing Kubernetes object corresponding
+                            to an Elastic resource managed by ECK.
+                          type: string
+                        namespace:
+                          description: Namespace of the Kubernetes object. If empty,
+                            defaults to the current namespace.
+                          type: string
+                        secretName:
+                          description: 'SecretName is the name of an existing Kubernetes
+                            secret that contains connection information for associating
+                            an Elastic resource not managed by the operator. The referenced
+                            secret must contain the following: - `url`: the URL to
+                            reach the Elastic resource - `username`: the username
+                            of the user to be authenticated to the Elastic resource
+                            - `password`: the password of the user to be authenticated
+                            to the Elastic resource - `ca.crt`: the CA certificate
+                            in PEM format (optional). This field cannot be used in
+                            combination with the other fields name, namespace or serviceName.'
+                          type: string
+                        serviceName:
+                          description: ServiceName is the name of an existing Kubernetes
+                            service which is used to make requests to the referenced
+                            object. It has to be in the same namespace as the referenced
+                            resource. If left empty, the default HTTP service of the
+                            referenced resource is used.
+                          type: string
+                      type: object
+                    type: array
+                type: object
               revisionHistoryLimit:
                 description: RevisionHistoryLimit is the number of revisions to retain
                   to allow rollback in the underlying DaemonSet or Deployment.
@@ -2432,6 +2480,15 @@ spec:
               kibanaAssociationStatus:
                 description: AssociationStatus is the status of an association resource.
                 type: string
+              monitoringAssociationStatus:
+                additionalProperties:
+                  description: AssociationStatus is the status of an association resource.
+                  type: string
+                description: AssociationStatusMap is the map of association's namespaced
+                  name string to its AssociationStatus. For resources that have a
+                  single Association of a given type (for ex. single ES reference),
+                  this map contains a single entry.
+                type: object
               observedGeneration:
                 description: ObservedGeneration represents the .metadata.generation
                   that the status is based upon. It corresponds to the metadata generation,

config/crds/v1/bases/beat.k8s.elastic.co_beats.yaml

@@ -15340,6 +15340,54 @@ spec:
                       is used.
                     type: string
                 type: object
+              monitoring:
+                description: Monitoring enables you to collect and ship monitoring
+                  data of this Beat. See https://www.elastic.co/guide/en/beats/filebeat/current/monitoring.html
+                  Internal Beat collectors are configured and send metrics data to
+                  one Elasticsearch monitoring cluster running in the same Kubernetes
+                  cluster.
+                properties:
+                  elasticsearchRefs:
+                    description: ElasticsearchRefs is a reference to a list of monitoring
+                      Elasticsearch clusters running in the same Kubernetes cluster.
+                      Due to existing limitations, only a single Elasticsearch cluster
+                      is currently supported.
+                    items:
+                      description: ObjectSelector defines a reference to a Kubernetes
+                        object which can be an Elastic resource managed by the operator
+                        or a Secret describing an external Elastic resource not managed
+                        by the operator.
+                      properties:
+                        name:
+                          description: Name of an existing Kubernetes object corresponding
+                            to an Elastic resource managed by ECK.
+                          type: string
+                        namespace:
+                          description: Namespace of the Kubernetes object. If empty,
+                            defaults to the current namespace.
+                          type: string
+                        secretName:
+                          description: 'SecretName is the name of an existing Kubernetes
+                            secret that contains connection information for associating
+                            an Elastic resource not managed by the operator. The referenced
+                            secret must contain the following: - `url`: the URL to
+                            reach the Elastic resource - `username`: the username
+                            of the user to be authenticated to the Elastic resource
+                            - `password`: the password of the user to be authenticated
+                            to the Elastic resource - `ca.crt`: the CA certificate
+                            in PEM format (optional). This field cannot be used in
+                            combination with the other fields name, namespace or serviceName.'
+                          type: string
+                        serviceName:
+                          description: ServiceName is the name of an existing Kubernetes
+                            service which is used to make requests to the referenced
+                            object. It has to be in the same namespace as the referenced
+                            resource. If left empty, the default HTTP service of the
+                            referenced resource is used.
+                          type: string
+                      type: object
+                    type: array
+                type: object
               revisionHistoryLimit:
                 description: RevisionHistoryLimit is the number of revisions to retain
                   to allow rollback in the underlying DaemonSet or Deployment.
@@ -15421,6 +15469,15 @@ spec:
               kibanaAssociationStatus:
                 description: AssociationStatus is the status of an association resource.
                 type: string
+              monitoringAssociationStatus:
+                additionalProperties:
+                  description: AssociationStatus is the status of an association resource.
+                  type: string
+                description: AssociationStatusMap is the map of association's namespaced
+                  name string to its AssociationStatus. For resources that have a
+                  single Association of a given type (for ex. single ES reference),
+                  this map contains a single entry.
+                type: object
               observedGeneration:
                 description: ObservedGeneration represents the .metadata.generation
                   that the status is based upon. It corresponds to the metadata generation,

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -2369,6 +2369,54 @@ spec:
                       is used.
                     type: string
                 type: object
+              monitoring:
+                description: Monitoring enables you to collect and ship monitoring
+                  data of this Beat. See https://www.elastic.co/guide/en/beats/filebeat/current/monitoring.html
+                  Internal Beat collectors are configured and send metrics data to
+                  one Elasticsearch monitoring cluster running in the same Kubernetes
+                  cluster.
+                properties:
+                  elasticsearchRefs:
+                    description: ElasticsearchRefs is a reference to a list of monitoring
+                      Elasticsearch clusters running in the same Kubernetes cluster.
+                      Due to existing limitations, only a single Elasticsearch cluster
+                      is currently supported.
+                    items:
+                      description: ObjectSelector defines a reference to a Kubernetes
+                        object which can be an Elastic resource managed by the operator
+                        or a Secret describing an external Elastic resource not managed
+                        by the operator.
+                      properties:
+                        name:
+                          description: Name of an existing Kubernetes object corresponding
+                            to an Elastic resource managed by ECK.
+                          type: string
+                        namespace:
+                          description: Namespace of the Kubernetes object. If empty,
+                            defaults to the current namespace.
+                          type: string
+                        secretName:
+                          description: 'SecretName is the name of an existing Kubernetes
+                            secret that contains connection information for associating
+                            an Elastic resource not managed by the operator. The referenced
+                            secret must contain the following: - `url`: the URL to
+                            reach the Elastic resource - `username`: the username
+                            of the user to be authenticated to the Elastic resource
+                            - `password`: the password of the user to be authenticated
+                            to the Elastic resource - `ca.crt`: the CA certificate
+                            in PEM format (optional). This field cannot be used in
+                            combination with the other fields name, namespace or serviceName.'
+                          type: string
+                        serviceName:
+                          description: ServiceName is the name of an existing Kubernetes
+                            service which is used to make requests to the referenced
+                            object. It has to be in the same namespace as the referenced
+                            resource. If left empty, the default HTTP service of the
+                            referenced resource is used.
+                          type: string
+                      type: object
+                    type: array
+                type: object
               revisionHistoryLimit:
                 description: RevisionHistoryLimit is the number of revisions to retain
                   to allow rollback in the underlying DaemonSet or Deployment.
@@ -2450,6 +2498,15 @@ spec:
               kibanaAssociationStatus:
                 description: AssociationStatus is the status of an association resource.
                 type: string
+              monitoringAssociationStatus:
+                additionalProperties:
+                  description: AssociationStatus is the status of an association resource.
+                  type: string
+                description: AssociationStatusMap is the map of association's namespaced
+                  name string to its AssociationStatus. For resources that have a
+                  single Association of a given type (for ex. single ES reference),
+                  this map contains a single entry.
+                type: object
               observedGeneration:
                 description: ObservedGeneration represents the .metadata.generation
                   that the status is based upon. It corresponds to the metadata generation,

docs/advanced-topics/stack-monitoring.asciidoc

@@ -9,7 +9,7 @@ endif::[]
 = Stack Monitoring
 
 You can enable link:https://www.elastic.co/guide/en/elasticsearch/reference/current/monitor-elasticsearch-cluster.html[Stack Monitoring]
-on Elasticsearch and Kibana to collect and ship their metrics and logs to a dedicated monitoring cluster.
+on Elasticsearch, Kibana and Beats to collect and ship their metrics and logs to a dedicated monitoring cluster.
 
 To enable stack monitoring, simply reference the monitoring Elasticsearch cluster in the `spec.monitoring` section of their specification.
 
@@ -57,10 +57,23 @@ spec:
       - name: monitoring
         namespace: observability <1>
   count: 1
+---
+apiVersion: beat.k8s.elastic.co/v1beta1
+kind: Beat
+metadata:
+  name: monitored-sample
+spec:
+  type: filebeat
+  version: {version}
+  monitoring:
+    elasticsearchRefs:
+    - name: monitoring
+      namespace: observability <3>
 ----
 
 <1> The use of `namespace` is optional if the monitoring Elasticsearch cluster and the monitored Elasticsearch cluster are running in the same namespace.
 <2> The use of `namespace` is optional if the Elasticsearch cluster and the Kibana instance are running in the same namespace.
+<3> The use of `namespace` is optional if the Elasticsearch cluster and the Beats instance are running in the same namespace.
 
 NOTE: You can configure an Elasticsearch cluster to monitor itself.
 

docs/reference/api-docs.asciidoc

@@ -315,6 +315,7 @@ BeatSpec defines the desired state of a Beat.
 | *`serviceAccountName`* __string__ | ServiceAccountName is used to check access from the current resource to Elasticsearch resource in a different namespace. Can only be used if ECK is enforcing RBAC on references.
 | *`daemonSet`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-beat-v1beta1-daemonsetspec[$$DaemonSetSpec$$]__ | DaemonSet specifies the Beat should be deployed as a DaemonSet, and allows providing its spec. Cannot be used along with `deployment`. If both are absent a default for the Type is used.
 | *`deployment`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-beat-v1beta1-deploymentspec[$$DeploymentSpec$$]__ | Deployment specifies the Beat should be deployed as a Deployment, and allows providing its spec. Cannot be used along with `daemonSet`. If both are absent a default for the Type is used.
+| *`monitoring`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-beat-v1beta1-monitoring[$$Monitoring$$]__ | Monitoring enables you to collect and ship monitoring data of this Beat. See https://www.elastic.co/guide/en/beats/filebeat/current/monitoring.html Internal Beat collectors are configured and send metrics data to one Elasticsearch monitoring cluster running in the same Kubernetes cluster.
 | *`revisionHistoryLimit`* __integer__ | RevisionHistoryLimit is the number of revisions to retain to allow rollback in the underlying DaemonSet or Deployment.
 |===
 
@@ -356,6 +357,23 @@ BeatSpec defines the desired state of a Beat.
 |===
 
 
+[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-beat-v1beta1-monitoring"]
+=== Monitoring 
+
+
+
+.Appears In:
+****
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-beat-v1beta1-beatspec[$$BeatSpec$$]
+****
+
+[cols="25a,75a", options="header"]
+|===
+| Field | Description
+| *`elasticsearchRefs`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-objectselector[$$ObjectSelector$$]__ | ElasticsearchRefs is a reference to a list of monitoring Elasticsearch clusters running in the same Kubernetes cluster. Due to existing limitations, only a single Elasticsearch cluster is currently supported.
+|===
+
+
 
 [id="{anchor_prefix}-common-k8s-elastic-co-v1"]
 == common.k8s.elastic.co/v1
@@ -488,6 +506,7 @@ ObjectSelector defines a reference to a Kubernetes object which can be an Elasti
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-maps-v1alpha1-mapsspec[$$MapsSpec$$]
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-metricsmonitoring[$$MetricsMonitoring$$]
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-kibana-v1-metricsmonitoring[$$MetricsMonitoring$$]
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-beat-v1beta1-monitoring[$$Monitoring$$]
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-agent-v1alpha1-output[$$Output$$]
 ****