Add cronjob metadata by default

CHANGELOG.next.asciidoc

@@ -88,6 +88,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 - Add FIPS configuration option for all AWS API calls. {pull}[28899]
 - Add support for non-unique Kafka headers for output messages. {pull}30369[30369]
 - Add action_input_type for the .fleet-actions-results {pull}30562[30562]
+- Add cronjob metadata by default {pull}30637[30637]
 
 *Auditbeat*
 

deploy/kubernetes/elastic-agent-managed-kubernetes.yaml

@@ -165,8 +165,7 @@ rules:
   - apiGroups: [ "batch" ]
     resources:
       - jobs
-      # Uncomment if need metadata for cronjob objects in versions >= v1.21
-      #- cronjobs
+      - cronjobs
     verbs: [ "get", "list", "watch" ]
   # required for apiserver
   - nonResourceURLs:

deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-role.yaml

@@ -38,8 +38,7 @@ rules:
   - apiGroups: [ "batch" ]
     resources:
       - jobs
-      # Uncomment if need metadata for cronjob objects in versions >= v1.21
-      #- cronjobs
+      - cronjobs
     verbs: [ "get", "list", "watch" ]
   # required for apiserver
   - nonResourceURLs:

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -703,8 +703,7 @@ rules:
   - apiGroups: ["batch"]
     resources:
       - jobs
-      # Uncomment if need metadata for cronjob objects in versions >= v1.21
-      #- cronjobs
+      - cronjobs
     verbs: ["get", "list", "watch"]
   - apiGroups:
       - ""

deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml

@@ -32,8 +32,7 @@ rules:
   - apiGroups: ["batch"]
     resources:
       - jobs
-      # Uncomment if need metadata for cronjob objects in versions >= v1.21
-      #- cronjobs
+      - cronjobs
     verbs: ["get", "list", "watch"]
   - apiGroups:
       - ""

deploy/kubernetes/filebeat-kubernetes.yaml

@@ -188,6 +188,10 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

deploy/kubernetes/filebeat/filebeat-role.yaml

@@ -18,6 +18,10 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

deploy/kubernetes/heartbeat-kubernetes.yaml

@@ -185,6 +185,10 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

deploy/kubernetes/heartbeat/heartbeat-role.yaml

@@ -16,6 +16,10 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -298,8 +298,7 @@ rules:
 - apiGroups: ["batch"]
   resources:
   - jobs
-  # Uncomment if need metadata for cronjob objects in versions >= v1.21
-  #- cronjobs
+  - cronjobs
   verbs: ["get", "list", "watch"]
 - apiGroups:
   - ""

deploy/kubernetes/metricbeat/metricbeat-role.yaml

@@ -31,8 +31,7 @@ rules:
 - apiGroups: ["batch"]
   resources:
   - jobs
-  # Uncomment if need metadata for cronjob objects in versions >= v1.21
-  #- cronjobs
+  - cronjobs
   verbs: ["get", "list", "watch"]
 - apiGroups:
   - ""

libbeat/common/kubernetes/metadata/config.go

@@ -35,6 +35,7 @@ type AddResourceMetadataConfig struct {
 	Node       *common.Config `config:"node"`
 	Namespace  *common.Config `config:"namespace"`
 	Deployment bool           `config:"deployment"`
+	CronJob    bool           `config:"cronjob"`
 }
 
 // InitDefaults initializes the defaults for the config.
@@ -56,5 +57,6 @@ func GetDefaultResourceMetadataConfig() *AddResourceMetadataConfig {
 		Node:       metaCfg,
 		Namespace:  metaCfg,
 		Deployment: true,
+		CronJob:    true,
 	}
 }

libbeat/common/kubernetes/metadata/pod.go

@@ -88,7 +88,6 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) common.
 	out := p.resource.GenerateK8s("pod", obj, opts...)
 
 	// check if Pod is handled by a ReplicaSet which is controlled by a Deployment
-	// TODO: same happens with CronJob vs Job. The hierarcy there is CronJob->Job->Pod
 	if p.addResourceMetadata.Deployment {
 		rsName, _ := out.GetValue("replicaset.name")
 		if rsName, ok := rsName.(string); ok {
@@ -99,6 +98,17 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) common.
 		}
 	}
 
+	// check if Pod is handled by a Job which is controlled by a CronJob
+	//if p.addResourceMetadata.CronJob {
+	//	jobName, _ := out.GetValue("job.name")
+	//	if jobName, ok := jobName.(string); ok {
+	//		dep := p.getCronjobOfJob(jobName, po.GetNamespace())
+	//		if dep != "" {
+	//			out.Put("cronjob.name", dep)
+	//		}
+	//	}
+	//}
+
 	if p.node != nil {
 		meta := p.node.GenerateFromName(po.Spec.NodeName, WithMetadata("node"))
 		if meta != nil {
@@ -162,3 +172,24 @@ func (p *pod) getRSDeployment(rsName string, ns string) string {
 	}
 	return ""
 }
+
+// getCronjobOfJob return the name of the Cronjob object that
+// owns the Job with the given name under the given Namespace
+func (p *pod) getCronjobOfJob(jobName string, ns string) string {
+	if p.client == nil {
+		return ""
+	}
+	cronjob, err := p.client.BatchV1().Jobs(ns).Get(context.TODO(), jobName, metav1.GetOptions{})
+	if err != nil {
+		return ""
+	}
+	for _, ref := range cronjob.GetOwnerReferences() {
+		if ref.Controller != nil && *ref.Controller {
+			switch ref.Kind {
+			case "CronJob":
+				return ref.Name
+			}
+		}
+	}
+	return ""
+}

libbeat/docs/shared-autodiscover.asciidoc

@@ -179,6 +179,8 @@ endif::[]
   storing requires special handling to avoid overloading the storage output. The enrichment of `node` or `namespace` metadata
   can be individually disabled by setting `enabled: false`. If resource is `pod` and it is created from a `deployment`, by default
   the deployment name is added, this can be disabled by set to `false`.
+  If resource is `pod` and it is created from a `cronjob`, by default
+  the cronjob name is added, this can be disabled by set to `false`.
   Example:
 
 ["source","yaml",subs="attributes"]
@@ -190,6 +192,7 @@ endif::[]
           include_labels: ["nodelabel2"]
           include_annotations: ["nodeannotation1"]
         deployment: false
+        cronjob: false
 -------------------------------------------------------------------------------------
 
 `unique`:: (Optional) Defaults to `false`. Marking an autodiscover provider as unique results into

metricbeat/docs/modules/kubernetes.asciidoc

@@ -241,6 +241,7 @@ metricbeat.modules:
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

metricbeat/metricbeat.reference.yml

@@ -510,6 +510,7 @@ metricbeat.modules:
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

metricbeat/module/kubernetes/_meta/config.reference.yml

@@ -33,6 +33,7 @@
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

metricbeat/module/kubernetes/_meta/config.yml

@@ -35,6 +35,7 @@
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

metricbeat/module/kubernetes/_meta/test/docs/02_objects/cronjob.yaml

@@ -16,7 +16,7 @@ spec:
             image: alpine
             imagePullPolicy: IfNotPresent
             command: ['sh', '-c', 'echo elastic world ; sleep 5']
-    
+
           restartPolicy: OnFailure
           terminationGracePeriodSeconds: 0
-  concurrencyPolicy: Allow
+  concurrencyPolicy: Allow

metricbeat/module/kubernetes/kubernetes.yml

@@ -111,20 +111,24 @@ spec:
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: basic-cronjob
+  name: hello
+  labels:
+    k8s-app: myjob
 spec:
-  schedule: "* * * * *"
+  schedule: "*/1 * * * *"
   jobTemplate:
     spec:
       template:
-        metadata:
-          name: basic-job
         spec:
           containers:
-          - name: hello
-            image: alpine:3
-            command: ["sh", "-c", "echo Hello!"]
-          restartPolicy: Never
+            - name: hello
+              image: busybox
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - -c
+                - date; echo Hello from the Kubernetes cluster
+          restartPolicy: never
 ---
 apiVersion: v1
 kind: ResourceQuota

metricbeat/module/kubernetes/state_cronjob/_meta/docs.asciidoc

@@ -1,6 +1,13 @@
 This is the `state_cronjob` metricset of the Kubernetes module.
 
-This metricset does not add metadata by default and hence in order to
-add metadata for this one need to configure the metricset with `add_metadata: true`
-and uncomment the proper `apiGroup` in the `ClusterRole`. Metadata are only available
-for versions of k8s >= v1.21.
+This metricset adds metadata by default only for versions of k8s >= v1.21.
+For older versions the APIs are not compatible and one need to configure the
+metricset with `add_metadata: false` and remove the proper `apiGroup` in the `ClusterRole`:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+- apiGroups: [ "batch" ]
+  resources:
+  - cronjobs
+-------------------------------------------------------------------------------------
+

metricbeat/module/kubernetes/state_cronjob/state_cronjob.go

@@ -90,6 +90,7 @@ func NewCronJobMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		ms.enricher = util.NewResourceMetadataEnricher(
 			base, &kubernetes.CronJob{}, false)
 	}
+
 	return &ms, nil
 }
 

metricbeat/modules.d/kubernetes.yml.disabled

@@ -38,6 +38,7 @@
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5

x-pack/metricbeat/metricbeat.reference.yml

@@ -898,6 +898,7 @@ metricbeat.modules:
   #     include_labels: ["nodelabel2"]
   #     include_annotations: ["nodeannotation1"]
   #   deployment: false
+  #   cronjob: false
   # Kubernetes client QPS and burst can be configured additionally
   #kube_client_options:
   #  qps: 5