fix: set cluster field in metadata pipelines

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Fixed
+
+- fix: set cluster field in metadata pipelines [#2284][#2284]
+
+[#2284]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2284
+[Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.8.0...main
+
 ## [v2.8.0]
 
 ### Released 2022-05-10

deploy/helm/sumologic/conf/events/events.conf

@@ -23,6 +23,13 @@
 <match fluentd.**>
   @type null
 </match>
+# Set `cluster` metadata field
+<filter **>
+  @type record_modifier
+  <record>
+    _sumo_metadata ${ {:fields => "cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}"} }
+  </record>
+</filter>
 # expose the Fluentd metrics to Prometheus
 <source>
   @type prometheus

deploy/helm/sumologic/conf/logs/fluentd/logs.source.containers.conf

@@ -68,6 +68,12 @@
     @include logs.kubernetes.sumologic.filter.conf
     exclude_namespace_regex {{ include "fluentd.excludeNamespaces" . }}
   </filter>
+  <filter **>
+    @type record_modifier
+    <record>
+      _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]}
+    </record>
+  </filter>
 {{- .Values.fluentd.logs.containers.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.containers.overrideOutputConf }}
   {{ .Values.fluentd.logs.containers.overrideOutputConf | nindent 4 }}

deploy/helm/sumologic/conf/logs/fluentd/logs.source.default.conf

@@ -23,6 +23,12 @@
   exclude_priority_regex {{ .Values.fluentd.logs.default.excludePriorityRegex | quote }}
   exclude_unit_regex {{ .Values.fluentd.logs.default.excludeUnitRegex | quote }}
 </filter>
+<filter **>
+  @type record_modifier
+  <record>
+    _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]}
+  </record>
+</filter>
 {{- .Values.fluentd.logs.default.extraOutputPluginConf | nindent 4 }}
 <match **>
   @type copy

deploy/helm/sumologic/conf/logs/fluentd/logs.source.systemd.conf

@@ -19,6 +19,12 @@
     exclude_priority_regex {{ .Values.fluentd.logs.kubelet.excludePriorityRegex | quote }}
     exclude_unit_regex {{ .Values.fluentd.logs.kubelet.excludeUnitRegex | quote }}
   </filter>
+  <filter **>
+    @type record_modifier
+    <record>
+      _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]}
+    </record>
+  </filter>
 {{- .Values.fluentd.logs.kubelet.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.kubelet.overrideOutputConf }}
   {{ .Values.fluentd.logs.kubelet.overrideOutputConf | nindent 4}}
@@ -78,6 +84,12 @@
       _sumo_metadata ${record["_sumo_metadata"][:source] = tag_parts[1]; record["_sumo_metadata"]}
     </record>
   </filter>
+  <filter **>
+    @type record_modifier
+    <record>
+      _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]}
+    </record>
+  </filter>
 {{- .Values.fluentd.logs.systemd.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.systemd.overrideOutputConf }}
   {{ .Values.fluentd.logs.systemd.overrideOutputConf | nindent 4 }}

deploy/helm/sumologic/conf/setup/resources.tf

@@ -2,7 +2,6 @@ resource "sumologic_collector" "collector" {
     name  = var.collector_name
     fields  = {
       {{- $fields := .Values.sumologic.collector.fields }}
-      {{ include "terraform.generate-key" (dict "Name" "cluster" "Value" "var.cluster_name" "SkipEscaping" true "KeyLength" (include "terraform.max-key-length" $fields)) }}
       {{- range $name, $value := $fields }}
       {{ include "terraform.generate-key" (dict "Name" $name "Value" $value "KeyLength" (include "terraform.max-key-length" $fields)) }}
       {{- end}}

deploy/helm/sumologic/conf/setup/variables.tf

@@ -1,8 +1,3 @@
-variable "cluster_name" {
-  type  = string
-  default = "{{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}"
-}
-
 variable "collector_name" {
   type  = string
   {{- if .Values.sumologic.collectorName }}

deploy/helm/sumologic/values.yaml

@@ -3453,7 +3453,6 @@ otelcol:
             ## extract the following well-known metadata fields
             - containerId
             - containerName
-            - clusterName
             - daemonSetName
             - deploymentName
             - hostName
@@ -3839,6 +3838,9 @@ metadata:
             - key: _origin  # add "_origin" metadata to metrics to keep the same format as for metrics from Fluentd
               value: kubernetes
               action: upsert
+            - key: cluster
+              value: '{{ .Values.sumologic.clusterName | quote }}'
+              action: upsert
         resource/remove_k8s_pod_pod_name:
           attributes:
             - action: delete
@@ -3923,7 +3925,6 @@ metadata:
           extract:
             metadata:
               ## extract the following well-known metadata fields
-              - clusterName
               - daemonSetName
               - deploymentName
               - nodeName
@@ -4169,6 +4170,11 @@ metadata:
           attributes:
             - pattern: ^pod_annotations_.*
               action: delete
+        resource/add_cluster:
+          attributes:
+            - key: cluster
+              value: '{{ .Values.sumologic.clusterName | quote }}'
+              action: upsert
         groupbyattrs/containers:
           keys:
             - k8s.container.id
@@ -4185,7 +4191,6 @@ metadata:
               ## extract the following well-known metadata fields
               - containerId
               - containerName
-              - clusterName
               - daemonSetName
               - deploymentName
               - hostName
@@ -4358,6 +4363,7 @@ metadata:
               - attributes/containers
               - groupbyattrs/containers
               - k8s_tagger
+              - resource/add_cluster
               - source/containers
               - resource/drop_annotations
               - attributes/remove_fluent_tag
@@ -4378,6 +4384,7 @@ metadata:
           #     - memory_limiter
           #     - groupbyattrs/containers
           #     - k8s_tagger
+          #     - resource/add_cluster
           #     - source/containers
           #     - resource/drop_annotations
           #     - resource/containers_copy_node_to_host
@@ -4397,6 +4404,7 @@ metadata:
               - filter/exclude_systemd_priority
               - filter/exclude_systemd_unit
               - groupbyattrs/systemd
+              - resource/add_cluster
               - source/systemd
               - attributes/remove_fluent_tag
               - batch
@@ -4414,6 +4422,7 @@ metadata:
               - filter/exclude_kubelet_priority
               - filter/exclude_kubelet_unit
               - groupbyattrs/systemd
+              - resource/add_cluster
               - source/kubelet
               - attributes/remove_fluent_tag
               - batch

tests/helm/events/static/apiServerUrl.output.yaml

@@ -22,6 +22,13 @@ data:
     <match fluentd.**>
       @type null
     </match>
+    # Set `cluster` metadata field
+    <filter **>
+      @type record_modifier
+      <record>
+        _sumo_metadata ${ {:fields => "cluster=kubernetes"} }
+      </record>
+    </filter>
     # expose the Fluentd metrics to Prometheus
     <source>
       @type prometheus

tests/helm/events/static/apiServerUrl_null.output.yaml

@@ -21,6 +21,13 @@ data:
     <match fluentd.**>
       @type null
     </match>
+    # Set `cluster` metadata field
+    <filter **>
+      @type record_modifier
+      <record>
+        _sumo_metadata ${ {:fields => "cluster=kubernetes"} }
+      </record>
+    </filter>
     # expose the Fluentd metrics to Prometheus
     <source>
       @type prometheus

tests/helm/events/static/default.output.yaml

@@ -21,6 +21,13 @@ data:
     <match fluentd.**>
       @type null
     </match>
+    # Set `cluster` metadata field
+    <filter **>
+      @type record_modifier
+      <record>
+        _sumo_metadata ${ {:fields => "cluster=kubernetes"} }
+      </record>
+    </filter>
     # expose the Fluentd metrics to Prometheus
     <source>
       @type prometheus

tests/helm/events/static/watchResourceEventsOverrides.output.yaml

@@ -29,6 +29,13 @@ data:
     <match fluentd.**>
       @type null
     </match>
+    # Set `cluster` metadata field
+    <filter **>
+      @type record_modifier
+      <record>
+        _sumo_metadata ${ {:fields => "cluster=kubernetes"} }
+      </record>
+    </filter>
     # expose the Fluentd metrics to Prometheus
     <source>
       @type prometheus

tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml

@@ -31,6 +31,13 @@ data:
     <match fluentd.**>
       @type null
     </match>
+    # Set `cluster` metadata field
+    <filter **>
+      @type record_modifier
+      <record>
+        _sumo_metadata ${ {:fields => "cluster=kubernetes"} }
+      </record>
+    </filter>
     # expose the Fluentd metrics to Prometheus
     <source>
       @type prometheus

tests/helm/metadata_logs_otc/static/basic.output.yaml

@@ -195,7 +195,6 @@ data:
           metadata:
           - containerId
           - containerName
-          - clusterName
           - daemonSetName
           - deploymentName
           - hostName
@@ -217,6 +216,11 @@ data:
         check_interval: 5s
         limit_percentage: 75
         spike_limit_percentage: 20
+      resource/add_cluster:
+        attributes:
+        - action: upsert
+          key: cluster
+          value: "kubernetes"
       resource/containers_copy_node_to_host:
         attributes:
         - action: upsert
@@ -281,6 +285,7 @@ data:
           - attributes/containers
           - groupbyattrs/containers
           - k8s_tagger
+          - resource/add_cluster
           - source/containers
           - resource/drop_annotations
           - attributes/remove_fluent_tag
@@ -300,6 +305,7 @@ data:
           - filter/exclude_kubelet_priority
           - filter/exclude_kubelet_unit
           - groupbyattrs/systemd
+          - resource/add_cluster
           - source/kubelet
           - attributes/remove_fluent_tag
           - batch
@@ -318,6 +324,7 @@ data:
           - filter/exclude_systemd_priority
           - filter/exclude_systemd_unit
           - groupbyattrs/systemd
+          - resource/add_cluster
           - source/systemd
           - attributes/remove_fluent_tag
           - batch

tests/helm/metadata_logs_otc/static/templates.output.yaml

@@ -195,7 +195,6 @@ data:
           metadata:
           - containerId
           - containerName
-          - clusterName
           - daemonSetName
           - deploymentName
           - hostName
@@ -217,6 +216,11 @@ data:
         check_interval: 5s
         limit_percentage: 75
         spike_limit_percentage: 20
+      resource/add_cluster:
+        attributes:
+        - action: upsert
+          key: cluster
+          value: "kubernetes"
       resource/containers_copy_node_to_host:
         attributes:
         - action: upsert
@@ -281,6 +285,7 @@ data:
           - attributes/containers
           - groupbyattrs/containers
           - k8s_tagger
+          - resource/add_cluster
           - source/containers
           - resource/drop_annotations
           - attributes/remove_fluent_tag
@@ -300,6 +305,7 @@ data:
           - filter/exclude_kubelet_priority
           - filter/exclude_kubelet_unit
           - groupbyattrs/systemd
+          - resource/add_cluster
           - source/kubelet
           - attributes/remove_fluent_tag
           - batch
@@ -318,6 +324,7 @@ data:
           - filter/exclude_systemd_priority
           - filter/exclude_systemd_unit
           - groupbyattrs/systemd
+          - resource/add_cluster
           - source/systemd
           - attributes/remove_fluent_tag
           - batch

tests/helm/terraform/static/all_fields.output.yaml

@@ -380,7 +380,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster = var.cluster_name
         }
     }
 
@@ -660,11 +659,6 @@ data:
 
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-
     variable "collector_name" {
       type  = string
       default = "kubernetes"

tests/helm/terraform/static/collector_fields.output.yaml

@@ -379,7 +379,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster                      = var.cluster_name
           here_is_very_long_field_name = "another_value"
           test_fields                  = "test_value"
         }
@@ -614,11 +613,6 @@ data:
 
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-
     variable "collector_name" {
       type  = string
       default = "kubernetes"

tests/helm/terraform/static/conditional_sources.output.yaml

@@ -369,7 +369,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster = var.cluster_name
         }
     }
 
@@ -532,11 +531,6 @@ data:
 
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-
     variable "collector_name" {
       type  = string
       default = "kubernetes"