fix: set cluster field in metadata pipelines

The `cluster` metadata field was set on the collector in Sumo. In the metadata pipelines, the behaviour was inconsistent. FluentD metrics set it correctly, but seemingly no other pipeline did. In particular, OT pipelines were actively deceptive, pretending to set it in the k8s_tagger processor, which doesn't actually do anything, and was recently deprecated upstream for this reason. Additionally, setting the field on the collector actually behaves subtly differently from sending it as a metric label, which caused one nasty bug for a client trialing OT. This change explicitly sets the field based on the configuration value and removes it from the collector.
SumoLogic · May 23, 2022 · e667c33 · e667c33
1 parent 7c3957d
commit e667c33
Show file tree

Hide file tree

Showing 28 changed files with 79 additions and 85 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Fixed
+
+- fix: set cluster field in metadata pipelines [#2284][#2284]
+
+[#2284]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2284
+[Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.8.0...main
+
 ## [v2.8.0]
 
 ### Released 2022-05-10

diff --git a/deploy/helm/sumologic/conf/events/events.conf b/deploy/helm/sumologic/conf/events/events.conf
@@ -23,6 +23,13 @@
 <match fluentd.**>
   @type null
 </match>
+# Set `cluster` metadata field
+<filter **>
+  @type record_modifier
+  <record>
+    _sumo_metadata ${ {:fields => "cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}"} }
+  </record>
+</filter>
 # expose the Fluentd metrics to Prometheus
 <source>
   @type prometheus

diff --git a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.containers.conf b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.containers.conf
@@ -68,6 +68,12 @@
     @include logs.kubernetes.sumologic.filter.conf
     exclude_namespace_regex {{ include "fluentd.excludeNamespaces" . }}
   </filter>
+  <filter **>
+    @type record_modifier
+    <record>
+      _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]}
+    </record>
+  </filter>
 {{- .Values.fluentd.logs.containers.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.containers.overrideOutputConf }}
   {{ .Values.fluentd.logs.containers.overrideOutputConf | nindent 4 }}

diff --git a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.default.conf b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.default.conf
@@ -23,6 +23,12 @@
   exclude_priority_regex {{ .Values.fluentd.logs.default.excludePriorityRegex | quote }}
   exclude_unit_regex {{ .Values.fluentd.logs.default.excludeUnitRegex | quote }}
 </filter>
+<filter **>
+  @type record_modifier
+  <record>
+    _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]}
+  </record>
+</filter>
 {{- .Values.fluentd.logs.default.extraOutputPluginConf | nindent 4 }}
 <match **>
   @type copy

diff --git a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.systemd.conf b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.systemd.conf
@@ -19,6 +19,12 @@
     exclude_priority_regex {{ .Values.fluentd.logs.kubelet.excludePriorityRegex | quote }}
     exclude_unit_regex {{ .Values.fluentd.logs.kubelet.excludeUnitRegex | quote }}
   </filter>
+  <filter **>
+    @type record_modifier
+    <record>
+      _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]}
+    </record>
+  </filter>
 {{- .Values.fluentd.logs.kubelet.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.kubelet.overrideOutputConf }}
   {{ .Values.fluentd.logs.kubelet.overrideOutputConf | nindent 4}}
@@ -78,6 +84,12 @@
       _sumo_metadata ${record["_sumo_metadata"][:source] = tag_parts[1]; record["_sumo_metadata"]}
     </record>
   </filter>
+  <filter **>
+    @type record_modifier
+    <record>
+      _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]}
+    </record>
+  </filter>
 {{- .Values.fluentd.logs.systemd.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.systemd.overrideOutputConf }}
   {{ .Values.fluentd.logs.systemd.overrideOutputConf | nindent 4 }}

diff --git a/deploy/helm/sumologic/conf/setup/resources.tf b/deploy/helm/sumologic/conf/setup/resources.tf
@@ -2,7 +2,6 @@ resource "sumologic_collector" "collector" {
     name  = var.collector_name
     fields  = {
       {{- $fields := .Values.sumologic.collector.fields }}
-      {{ include "terraform.generate-key" (dict "Name" "cluster" "Value" "var.cluster_name" "SkipEscaping" true "KeyLength" (include "terraform.max-key-length" $fields)) }}
       {{- range $name, $value := $fields }}
       {{ include "terraform.generate-key" (dict "Name" $name "Value" $value "KeyLength" (include "terraform.max-key-length" $fields)) }}
       {{- end}}

diff --git a/deploy/helm/sumologic/conf/setup/variables.tf b/deploy/helm/sumologic/conf/setup/variables.tf
@@ -1,8 +1,3 @@
-variable "cluster_name" {
-  type  = string
-  default = "{{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}"
-}
-
 variable "collector_name" {
   type  = string
   {{- if .Values.sumologic.collectorName }}

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -3453,7 +3453,6 @@ otelcol:
             ## extract the following well-known metadata fields
             - containerId
             - containerName
-            - clusterName
             - daemonSetName
             - deploymentName
             - hostName
@@ -3839,6 +3838,9 @@ metadata:
             - key: _origin  # add "_origin" metadata to metrics to keep the same format as for metrics from Fluentd
               value: kubernetes
               action: upsert
+            - key: cluster
+              value: '{{ .Values.sumologic.clusterName | quote }}'
+              action: upsert
         resource/remove_k8s_pod_pod_name:
           attributes:
             - action: delete
@@ -3923,7 +3925,6 @@ metadata:
           extract:
             metadata:
               ## extract the following well-known metadata fields
-              - clusterName
               - daemonSetName
               - deploymentName
               - nodeName
@@ -4169,6 +4170,11 @@ metadata:
           attributes:
             - pattern: ^pod_annotations_.*
               action: delete
+        resource/add_cluster:
+          attributes:
+            - key: cluster
+              value: '{{ .Values.sumologic.clusterName | quote }}'
+              action: upsert
         groupbyattrs/containers:
           keys:
             - k8s.container.id
@@ -4185,7 +4191,6 @@ metadata:
               ## extract the following well-known metadata fields
               - containerId
               - containerName
-              - clusterName
               - daemonSetName
               - deploymentName
               - hostName
@@ -4358,6 +4363,7 @@ metadata:
               - attributes/containers
               - groupbyattrs/containers
               - k8s_tagger
+              - resource/add_cluster
               - source/containers
               - resource/drop_annotations
               - attributes/remove_fluent_tag
@@ -4378,6 +4384,7 @@ metadata:
           #     - memory_limiter
           #     - groupbyattrs/containers
           #     - k8s_tagger
+          #     - resource/add_cluster
           #     - source/containers
           #     - resource/drop_annotations
           #     - resource/containers_copy_node_to_host
@@ -4397,6 +4404,7 @@ metadata:
               - filter/exclude_systemd_priority
               - filter/exclude_systemd_unit
               - groupbyattrs/systemd
+              - resource/add_cluster
               - source/systemd
               - attributes/remove_fluent_tag
               - batch
@@ -4414,6 +4422,7 @@ metadata:
               - filter/exclude_kubelet_priority
               - filter/exclude_kubelet_unit
               - groupbyattrs/systemd
+              - resource/add_cluster
               - source/kubelet
               - attributes/remove_fluent_tag
               - batch

diff --git a/tests/helm/metadata_logs_otc/static/basic.output.yaml b/tests/helm/metadata_logs_otc/static/basic.output.yaml
@@ -195,7 +195,6 @@ data:
           metadata:
           - containerId
           - containerName
-          - clusterName
           - daemonSetName
           - deploymentName
           - hostName
@@ -217,6 +216,11 @@ data:
         check_interval: 5s
         limit_percentage: 75
         spike_limit_percentage: 20
+      resource/add_cluster:
+        attributes:
+        - action: upsert
+          key: cluster
+          value: "kubernetes"
       resource/containers_copy_node_to_host:
         attributes:
         - action: upsert
@@ -281,6 +285,7 @@ data:
           - attributes/containers
           - groupbyattrs/containers
           - k8s_tagger
+          - resource/add_cluster
           - source/containers
           - resource/drop_annotations
           - attributes/remove_fluent_tag
@@ -300,6 +305,7 @@ data:
           - filter/exclude_kubelet_priority
           - filter/exclude_kubelet_unit
           - groupbyattrs/systemd
+          - resource/add_cluster
           - source/kubelet
           - attributes/remove_fluent_tag
           - batch
@@ -318,6 +324,7 @@ data:
           - filter/exclude_systemd_priority
           - filter/exclude_systemd_unit
           - groupbyattrs/systemd
+          - resource/add_cluster
           - source/systemd
           - attributes/remove_fluent_tag
           - batch

diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml
@@ -195,7 +195,6 @@ data:
           metadata:
           - containerId
           - containerName
-          - clusterName
           - daemonSetName
           - deploymentName
           - hostName
@@ -217,6 +216,11 @@ data:
         check_interval: 5s
         limit_percentage: 75
         spike_limit_percentage: 20
+      resource/add_cluster:
+        attributes:
+        - action: upsert
+          key: cluster
+          value: "kubernetes"
       resource/containers_copy_node_to_host:
         attributes:
         - action: upsert
@@ -281,6 +285,7 @@ data:
           - attributes/containers
           - groupbyattrs/containers
           - k8s_tagger
+          - resource/add_cluster
           - source/containers
           - resource/drop_annotations
           - attributes/remove_fluent_tag
@@ -300,6 +305,7 @@ data:
           - filter/exclude_kubelet_priority
           - filter/exclude_kubelet_unit
           - groupbyattrs/systemd
+          - resource/add_cluster
           - source/kubelet
           - attributes/remove_fluent_tag
           - batch
@@ -318,6 +324,7 @@ data:
           - filter/exclude_systemd_priority
           - filter/exclude_systemd_unit
           - groupbyattrs/systemd
+          - resource/add_cluster
           - source/systemd
           - attributes/remove_fluent_tag
           - batch

diff --git a/tests/helm/terraform/static/all_fields.output.yaml b/tests/helm/terraform/static/all_fields.output.yaml
@@ -380,7 +380,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster = var.cluster_name
         }
     }
     
@@ -660,11 +659,6 @@ data:
   
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-  
     variable "collector_name" {
       type  = string
       default = "kubernetes"

diff --git a/tests/helm/terraform/static/collector_fields.output.yaml b/tests/helm/terraform/static/collector_fields.output.yaml
@@ -379,7 +379,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster                      = var.cluster_name
           here_is_very_long_field_name = "another_value"
           test_fields                  = "test_value"
         }
@@ -614,11 +613,6 @@ data:
   
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-  
     variable "collector_name" {
       type  = string
       default = "kubernetes"

diff --git a/tests/helm/terraform/static/conditional_sources.output.yaml b/tests/helm/terraform/static/conditional_sources.output.yaml
@@ -369,7 +369,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster = var.cluster_name
         }
     }
   
@@ -532,11 +531,6 @@ data:
   
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-  
     variable "collector_name" {
       type  = string
       default = "kubernetes"

diff --git a/tests/helm/terraform/static/custom.output.yaml b/tests/helm/terraform/static/custom.output.yaml
@@ -369,7 +369,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster = var.cluster_name
         }
     }
   
@@ -532,11 +531,6 @@ data:
   
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-  
     variable "collector_name" {
       type  = string
       default = "kubernetes"

diff --git a/tests/helm/terraform/static/default.output.yaml b/tests/helm/terraform/static/default.output.yaml
@@ -379,7 +379,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster = var.cluster_name
         }
     }
     
@@ -612,11 +611,6 @@ data:
   
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-  
     variable "collector_name" {
       type  = string
       default = "kubernetes"

diff --git a/tests/helm/terraform/static/disable_default_metrics.output.yaml b/tests/helm/terraform/static/disable_default_metrics.output.yaml
@@ -378,7 +378,6 @@ data:
     resource "sumologic_collector" "collector" {
         name  = var.collector_name
         fields  = {
-          cluster = var.cluster_name
         }
     }
     
@@ -604,11 +603,6 @@ data:
   
     bash /etc/terraform/custom.sh
   variables.tf: |
-    variable "cluster_name" {
-      type  = string
-      default = "kubernetes"
-    }
-  
     variable "collector_name" {
       type  = string
       default = "kubernetes"