From b48baa0eb3407bc39c13c1559ea30e6148372ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Tue, 17 May 2022 12:46:52 +0200 Subject: [PATCH] fix: set cluster field in metadata pipelines The `cluster` metadata field was set on the collector in Sumo. In the metadata pipelines, the behaviour was inconsistent. FluentD metrics set it correctly, but seemingly no other pipeline did. In particular, OT pipelines were actively deceptive, pretending to set it in the k8s_tagger processor, which doesn't actually do anything, and was recently deprecated upstream for this reason. Additionally, setting the field on the collector actually behaves subtly differently from sending it as a metric label, which caused one nasty bug for a client trialing OT. This change explicitly sets the field based on the configuration value and removes it from the collector. --- CHANGELOG.md | 9 +++++++++ deploy/helm/sumologic/conf/events/events.conf | 7 +++++++ .../conf/logs/fluentd/logs.source.containers.conf | 6 ++++++ .../conf/logs/fluentd/logs.source.default.conf | 6 ++++++ .../conf/logs/fluentd/logs.source.systemd.conf | 12 ++++++++++++ deploy/helm/sumologic/conf/setup/resources.tf | 1 - deploy/helm/sumologic/conf/setup/variables.tf | 5 ----- deploy/helm/sumologic/values.yaml | 15 ++++++++++++--- tests/helm/events/static/apiServerUrl.output.yaml | 7 +++++++ .../events/static/apiServerUrl_null.output.yaml | 7 +++++++ tests/helm/events/static/default.output.yaml | 7 +++++++ .../watchResourceEventsOverrides.output.yaml | 7 +++++++ ...sourceEventsOverrides_apiServerUrl.output.yaml | 7 +++++++ .../metadata_logs_otc/static/basic.output.yaml | 9 ++++++++- .../static/templates.output.yaml | 9 ++++++++- .../helm/terraform/static/all_fields.output.yaml | 6 ------ .../terraform/static/collector_fields.output.yaml | 6 ------ .../static/conditional_sources.output.yaml | 6 ------ tests/helm/terraform/static/custom.output.yaml | 6 ------ tests/helm/terraform/static/default.output.yaml | 6 ------ .../static/disable_default_metrics.output.yaml | 6 ------ .../static/disabled_dashboards.output.yaml | 6 ------ .../static/disabled_monitors.output.yaml | 6 ------ .../monitors_with_email_notifications.output.yaml | 6 ------ .../static/monitors_with_single_email.output.yaml | 6 ------ .../static/strip_extrapolation.output.yaml | 6 ------ tests/helm/terraform/static/traces.output.yaml | 6 ------ .../collection-monitoring-false.output.yaml | 1 - tests/helm/tracing/static/simple.output.yaml | 1 - .../integration/helm_default_installation_test.go | 5 +++++ .../helm_otc_metadata_installation_test.go | 4 ++++ tests/integration/helm_otelcol_logs_test.go | 1 + .../values/values_helm_otelcol_logs.yaml | 1 + 33 files changed, 114 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b0099d15c..861889bfc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- fix: set cluster field in metadata pipelines [#2284][#2284] + +[#2284]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2284 +[Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.8.0...main + ## [v2.8.0] ### Released 2022-05-10 diff --git a/deploy/helm/sumologic/conf/events/events.conf b/deploy/helm/sumologic/conf/events/events.conf index e124ca5c06..024e99fb0b 100644 --- a/deploy/helm/sumologic/conf/events/events.conf +++ b/deploy/helm/sumologic/conf/events/events.conf @@ -23,6 +23,13 @@ @type null +# Set `cluster` metadata field + + @type record_modifier + + _sumo_metadata ${ {:fields => "cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}"} } + + # expose the Fluentd metrics to Prometheus @type prometheus diff --git a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.containers.conf b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.containers.conf index 9ea0b0bd34..d41ba3c9ad 100644 --- a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.containers.conf +++ b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.containers.conf @@ -68,6 +68,12 @@ @include logs.kubernetes.sumologic.filter.conf exclude_namespace_regex {{ include "fluentd.excludeNamespaces" . }} + + @type record_modifier + + _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]} + + {{- .Values.fluentd.logs.containers.extraOutputPluginConf | nindent 4 }} {{ if .Values.fluentd.logs.containers.overrideOutputConf }} {{ .Values.fluentd.logs.containers.overrideOutputConf | nindent 4 }} diff --git a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.default.conf b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.default.conf index 67054d92c5..542199085d 100644 --- a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.default.conf +++ b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.default.conf @@ -23,6 +23,12 @@ exclude_priority_regex {{ .Values.fluentd.logs.default.excludePriorityRegex | quote }} exclude_unit_regex {{ .Values.fluentd.logs.default.excludeUnitRegex | quote }} + + @type record_modifier + + _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]} + + {{- .Values.fluentd.logs.default.extraOutputPluginConf | nindent 4 }} @type copy diff --git a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.systemd.conf b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.systemd.conf index 8bfc4e5690..513c4796cf 100644 --- a/deploy/helm/sumologic/conf/logs/fluentd/logs.source.systemd.conf +++ b/deploy/helm/sumologic/conf/logs/fluentd/logs.source.systemd.conf @@ -19,6 +19,12 @@ exclude_priority_regex {{ .Values.fluentd.logs.kubelet.excludePriorityRegex | quote }} exclude_unit_regex {{ .Values.fluentd.logs.kubelet.excludeUnitRegex | quote }} + + @type record_modifier + + _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]} + + {{- .Values.fluentd.logs.kubelet.extraOutputPluginConf | nindent 4 }} {{ if .Values.fluentd.logs.kubelet.overrideOutputConf }} {{ .Values.fluentd.logs.kubelet.overrideOutputConf | nindent 4}} @@ -78,6 +84,12 @@ _sumo_metadata ${record["_sumo_metadata"][:source] = tag_parts[1]; record["_sumo_metadata"]} + + @type record_modifier + + _sumo_metadata ${record["_sumo_metadata"][:fields] = record["_sumo_metadata"].fetch(:fields, "").split(",").append("cluster={{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}").join(","); record["_sumo_metadata"]} + + {{- .Values.fluentd.logs.systemd.extraOutputPluginConf | nindent 4 }} {{ if .Values.fluentd.logs.systemd.overrideOutputConf }} {{ .Values.fluentd.logs.systemd.overrideOutputConf | nindent 4 }} diff --git a/deploy/helm/sumologic/conf/setup/resources.tf b/deploy/helm/sumologic/conf/setup/resources.tf index 81195e3698..357c413799 100644 --- a/deploy/helm/sumologic/conf/setup/resources.tf +++ b/deploy/helm/sumologic/conf/setup/resources.tf @@ -2,7 +2,6 @@ resource "sumologic_collector" "collector" { name = var.collector_name fields = { {{- $fields := .Values.sumologic.collector.fields }} - {{ include "terraform.generate-key" (dict "Name" "cluster" "Value" "var.cluster_name" "SkipEscaping" true "KeyLength" (include "terraform.max-key-length" $fields)) }} {{- range $name, $value := $fields }} {{ include "terraform.generate-key" (dict "Name" $name "Value" $value "KeyLength" (include "terraform.max-key-length" $fields)) }} {{- end}} diff --git a/deploy/helm/sumologic/conf/setup/variables.tf b/deploy/helm/sumologic/conf/setup/variables.tf index cf084b3532..4d9baeb67e 100644 --- a/deploy/helm/sumologic/conf/setup/variables.tf +++ b/deploy/helm/sumologic/conf/setup/variables.tf @@ -1,8 +1,3 @@ -variable "cluster_name" { - type = string - default = "{{ template "sumologic.clusterNameReplaceSpaceWithDash" . }}" -} - variable "collector_name" { type = string {{- if .Values.sumologic.collectorName }} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 86bbb9f190..12179a67a8 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3453,7 +3453,6 @@ otelcol: ## extract the following well-known metadata fields - containerId - containerName - - clusterName - daemonSetName - deploymentName - hostName @@ -3839,6 +3838,9 @@ metadata: - key: _origin # add "_origin" metadata to metrics to keep the same format as for metrics from Fluentd value: kubernetes action: upsert + - key: cluster + value: '{{ .Values.sumologic.clusterName | quote }}' + action: upsert resource/remove_k8s_pod_pod_name: attributes: - action: delete @@ -3923,7 +3925,6 @@ metadata: extract: metadata: ## extract the following well-known metadata fields - - clusterName - daemonSetName - deploymentName - nodeName @@ -4169,6 +4170,11 @@ metadata: attributes: - pattern: ^pod_annotations_.* action: delete + resource/add_cluster: + attributes: + - key: cluster + value: '{{ .Values.sumologic.clusterName | quote }}' + action: upsert groupbyattrs/containers: keys: - k8s.container.id @@ -4185,7 +4191,6 @@ metadata: ## extract the following well-known metadata fields - containerId - containerName - - clusterName - daemonSetName - deploymentName - hostName @@ -4358,6 +4363,7 @@ metadata: - attributes/containers - groupbyattrs/containers - k8s_tagger + - resource/add_cluster - source/containers - resource/drop_annotations - attributes/remove_fluent_tag @@ -4378,6 +4384,7 @@ metadata: # - memory_limiter # - groupbyattrs/containers # - k8s_tagger + # - resource/add_cluster # - source/containers # - resource/drop_annotations # - resource/containers_copy_node_to_host @@ -4397,6 +4404,7 @@ metadata: - filter/exclude_systemd_priority - filter/exclude_systemd_unit - groupbyattrs/systemd + - resource/add_cluster - source/systemd - attributes/remove_fluent_tag - batch @@ -4414,6 +4422,7 @@ metadata: - filter/exclude_kubelet_priority - filter/exclude_kubelet_unit - groupbyattrs/systemd + - resource/add_cluster - source/kubelet - attributes/remove_fluent_tag - batch diff --git a/tests/helm/events/static/apiServerUrl.output.yaml b/tests/helm/events/static/apiServerUrl.output.yaml index aff672c625..cf4a859cbb 100644 --- a/tests/helm/events/static/apiServerUrl.output.yaml +++ b/tests/helm/events/static/apiServerUrl.output.yaml @@ -22,6 +22,13 @@ data: @type null + # Set `cluster` metadata field + + @type record_modifier + + _sumo_metadata ${ {:fields => "cluster=kubernetes"} } + + # expose the Fluentd metrics to Prometheus @type prometheus diff --git a/tests/helm/events/static/apiServerUrl_null.output.yaml b/tests/helm/events/static/apiServerUrl_null.output.yaml index 2e01890f88..97a6a11431 100644 --- a/tests/helm/events/static/apiServerUrl_null.output.yaml +++ b/tests/helm/events/static/apiServerUrl_null.output.yaml @@ -21,6 +21,13 @@ data: @type null + # Set `cluster` metadata field + + @type record_modifier + + _sumo_metadata ${ {:fields => "cluster=kubernetes"} } + + # expose the Fluentd metrics to Prometheus @type prometheus diff --git a/tests/helm/events/static/default.output.yaml b/tests/helm/events/static/default.output.yaml index 2e01890f88..97a6a11431 100644 --- a/tests/helm/events/static/default.output.yaml +++ b/tests/helm/events/static/default.output.yaml @@ -21,6 +21,13 @@ data: @type null + # Set `cluster` metadata field + + @type record_modifier + + _sumo_metadata ${ {:fields => "cluster=kubernetes"} } + + # expose the Fluentd metrics to Prometheus @type prometheus diff --git a/tests/helm/events/static/watchResourceEventsOverrides.output.yaml b/tests/helm/events/static/watchResourceEventsOverrides.output.yaml index b679f7a16d..b340a95099 100644 --- a/tests/helm/events/static/watchResourceEventsOverrides.output.yaml +++ b/tests/helm/events/static/watchResourceEventsOverrides.output.yaml @@ -29,6 +29,13 @@ data: @type null + # Set `cluster` metadata field + + @type record_modifier + + _sumo_metadata ${ {:fields => "cluster=kubernetes"} } + + # expose the Fluentd metrics to Prometheus @type prometheus diff --git a/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml b/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml index 0c96237b38..0dbb40e9e2 100644 --- a/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml +++ b/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml @@ -31,6 +31,13 @@ data: @type null + # Set `cluster` metadata field + + @type record_modifier + + _sumo_metadata ${ {:fields => "cluster=kubernetes"} } + + # expose the Fluentd metrics to Prometheus @type prometheus diff --git a/tests/helm/metadata_logs_otc/static/basic.output.yaml b/tests/helm/metadata_logs_otc/static/basic.output.yaml index 840a8b9f92..ddc6204b6f 100644 --- a/tests/helm/metadata_logs_otc/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc/static/basic.output.yaml @@ -195,7 +195,6 @@ data: metadata: - containerId - containerName - - clusterName - daemonSetName - deploymentName - hostName @@ -217,6 +216,11 @@ data: check_interval: 5s limit_percentage: 75 spike_limit_percentage: 20 + resource/add_cluster: + attributes: + - action: upsert + key: cluster + value: "kubernetes" resource/containers_copy_node_to_host: attributes: - action: upsert @@ -281,6 +285,7 @@ data: - attributes/containers - groupbyattrs/containers - k8s_tagger + - resource/add_cluster - source/containers - resource/drop_annotations - attributes/remove_fluent_tag @@ -300,6 +305,7 @@ data: - filter/exclude_kubelet_priority - filter/exclude_kubelet_unit - groupbyattrs/systemd + - resource/add_cluster - source/kubelet - attributes/remove_fluent_tag - batch @@ -318,6 +324,7 @@ data: - filter/exclude_systemd_priority - filter/exclude_systemd_unit - groupbyattrs/systemd + - resource/add_cluster - source/systemd - attributes/remove_fluent_tag - batch diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml index cc7efd30e6..48d6d83d43 100644 --- a/tests/helm/metadata_logs_otc/static/templates.output.yaml +++ b/tests/helm/metadata_logs_otc/static/templates.output.yaml @@ -195,7 +195,6 @@ data: metadata: - containerId - containerName - - clusterName - daemonSetName - deploymentName - hostName @@ -217,6 +216,11 @@ data: check_interval: 5s limit_percentage: 75 spike_limit_percentage: 20 + resource/add_cluster: + attributes: + - action: upsert + key: cluster + value: "kubernetes" resource/containers_copy_node_to_host: attributes: - action: upsert @@ -281,6 +285,7 @@ data: - attributes/containers - groupbyattrs/containers - k8s_tagger + - resource/add_cluster - source/containers - resource/drop_annotations - attributes/remove_fluent_tag @@ -300,6 +305,7 @@ data: - filter/exclude_kubelet_priority - filter/exclude_kubelet_unit - groupbyattrs/systemd + - resource/add_cluster - source/kubelet - attributes/remove_fluent_tag - batch @@ -318,6 +324,7 @@ data: - filter/exclude_systemd_priority - filter/exclude_systemd_unit - groupbyattrs/systemd + - resource/add_cluster - source/systemd - attributes/remove_fluent_tag - batch diff --git a/tests/helm/terraform/static/all_fields.output.yaml b/tests/helm/terraform/static/all_fields.output.yaml index 636d3e4bc6..d50d896abd 100644 --- a/tests/helm/terraform/static/all_fields.output.yaml +++ b/tests/helm/terraform/static/all_fields.output.yaml @@ -380,7 +380,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -660,11 +659,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/collector_fields.output.yaml b/tests/helm/terraform/static/collector_fields.output.yaml index 85ef518b01..79ff894250 100644 --- a/tests/helm/terraform/static/collector_fields.output.yaml +++ b/tests/helm/terraform/static/collector_fields.output.yaml @@ -379,7 +379,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name here_is_very_long_field_name = "another_value" test_fields = "test_value" } @@ -614,11 +613,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/conditional_sources.output.yaml b/tests/helm/terraform/static/conditional_sources.output.yaml index b3c585aac6..1a23c85d67 100644 --- a/tests/helm/terraform/static/conditional_sources.output.yaml +++ b/tests/helm/terraform/static/conditional_sources.output.yaml @@ -369,7 +369,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -532,11 +531,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/custom.output.yaml b/tests/helm/terraform/static/custom.output.yaml index b3c585aac6..1a23c85d67 100644 --- a/tests/helm/terraform/static/custom.output.yaml +++ b/tests/helm/terraform/static/custom.output.yaml @@ -369,7 +369,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -532,11 +531,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/default.output.yaml b/tests/helm/terraform/static/default.output.yaml index a27c7e0dbe..ef0d9a1468 100644 --- a/tests/helm/terraform/static/default.output.yaml +++ b/tests/helm/terraform/static/default.output.yaml @@ -379,7 +379,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -612,11 +611,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/disable_default_metrics.output.yaml b/tests/helm/terraform/static/disable_default_metrics.output.yaml index bfd0b4beb3..e1697ac911 100644 --- a/tests/helm/terraform/static/disable_default_metrics.output.yaml +++ b/tests/helm/terraform/static/disable_default_metrics.output.yaml @@ -378,7 +378,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -604,11 +603,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/disabled_dashboards.output.yaml b/tests/helm/terraform/static/disabled_dashboards.output.yaml index a6454ba6a3..f0f989da58 100644 --- a/tests/helm/terraform/static/disabled_dashboards.output.yaml +++ b/tests/helm/terraform/static/disabled_dashboards.output.yaml @@ -379,7 +379,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -614,11 +613,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/disabled_monitors.output.yaml b/tests/helm/terraform/static/disabled_monitors.output.yaml index f92dded490..fdd39912d4 100644 --- a/tests/helm/terraform/static/disabled_monitors.output.yaml +++ b/tests/helm/terraform/static/disabled_monitors.output.yaml @@ -379,7 +379,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -614,11 +613,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/monitors_with_email_notifications.output.yaml b/tests/helm/terraform/static/monitors_with_email_notifications.output.yaml index fcca57c65a..faa8347e8b 100644 --- a/tests/helm/terraform/static/monitors_with_email_notifications.output.yaml +++ b/tests/helm/terraform/static/monitors_with_email_notifications.output.yaml @@ -385,7 +385,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -618,11 +617,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/monitors_with_single_email.output.yaml b/tests/helm/terraform/static/monitors_with_single_email.output.yaml index 039e619529..a0dad022aa 100644 --- a/tests/helm/terraform/static/monitors_with_single_email.output.yaml +++ b/tests/helm/terraform/static/monitors_with_single_email.output.yaml @@ -385,7 +385,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -618,11 +617,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/strip_extrapolation.output.yaml b/tests/helm/terraform/static/strip_extrapolation.output.yaml index d688862e3f..9b0ebabfec 100644 --- a/tests/helm/terraform/static/strip_extrapolation.output.yaml +++ b/tests/helm/terraform/static/strip_extrapolation.output.yaml @@ -380,7 +380,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -613,11 +612,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/terraform/static/traces.output.yaml b/tests/helm/terraform/static/traces.output.yaml index 6dd6bb74da..0b72c23212 100644 --- a/tests/helm/terraform/static/traces.output.yaml +++ b/tests/helm/terraform/static/traces.output.yaml @@ -371,7 +371,6 @@ data: resource "sumologic_collector" "collector" { name = var.collector_name fields = { - cluster = var.cluster_name } } @@ -548,11 +547,6 @@ data: bash /etc/terraform/custom.sh variables.tf: | - variable "cluster_name" { - type = string - default = "kubernetes" - } - variable "collector_name" { type = string default = "kubernetes" diff --git a/tests/helm/tracing/static/collection-monitoring-false.output.yaml b/tests/helm/tracing/static/collection-monitoring-false.output.yaml index 1c7c95c01f..df63737039 100644 --- a/tests/helm/tracing/static/collection-monitoring-false.output.yaml +++ b/tests/helm/tracing/static/collection-monitoring-false.output.yaml @@ -58,7 +58,6 @@ data: metadata: - containerId - containerName - - clusterName - daemonSetName - deploymentName - hostName diff --git a/tests/helm/tracing/static/simple.output.yaml b/tests/helm/tracing/static/simple.output.yaml index 47ca46dc3d..990f2516a0 100644 --- a/tests/helm/tracing/static/simple.output.yaml +++ b/tests/helm/tracing/static/simple.output.yaml @@ -58,7 +58,6 @@ data: metadata: - containerId - containerName - - clusterName - daemonSetName - deploymentName - hostName diff --git a/tests/integration/helm_default_installation_test.go b/tests/integration/helm_default_installation_test.go index e60b075c85..e5144b3a23 100644 --- a/tests/integration/helm_default_installation_test.go +++ b/tests/integration/helm_default_installation_test.go @@ -240,6 +240,7 @@ func Test_Helm_Default_FluentD_Metadata(t *testing.T) { labels := sample.Labels expectedLabels := receivermock.Labels{ "_origin": "kubernetes", + "cluster": "kubernetes", "container": "receiver-mock", // TODO: figure out why is this flaky and sometimes it's not there // https://github.com/SumoLogic/sumologic-kubernetes-collection/runs/4508796836?check_suite_focus=true @@ -299,6 +300,7 @@ func Test_Helm_Default_FluentD_Metadata(t *testing.T) { Assess("expected container log metadata is present", stepfuncs.WaitUntilExpectedLogsPresent( logsGeneratorCount, map[string]string{ + "cluster": "kubernetes", "namespace": internal.LogsGeneratorName, "pod_labels_app": internal.LogsGeneratorName, "container": internal.LogsGeneratorName, @@ -324,6 +326,7 @@ func Test_Helm_Default_FluentD_Metadata(t *testing.T) { Assess("logs from node systemd present", stepfuncs.WaitUntilExpectedLogsPresent( 10, // we don't really control this, just want to check if the logs show up map[string]string{ + "cluster": "kubernetes", "_sourceName": "", "_sourceCategory": "kubernetes/system", "_sourceHost": "", @@ -337,6 +340,7 @@ func Test_Helm_Default_FluentD_Metadata(t *testing.T) { Assess("logs from kubelet present", stepfuncs.WaitUntilExpectedLogsPresent( 1, // we don't really control this, just want to check if the logs show up map[string]string{ + "cluster": "kubernetes", "_sourceName": "k8s_kubelet", "_sourceCategory": "kubernetes/kubelet", "_sourceHost": "", @@ -363,6 +367,7 @@ func Test_Helm_Default_FluentD_Metadata(t *testing.T) { map[string]string{ "_sourceName": "events", "_sourceCategory": fmt.Sprintf("%s/events", internal.ClusterName), + "cluster": "kubernetes", }, internal.ReceiverMockNamespace, internal.ReceiverMockServiceName, diff --git a/tests/integration/helm_otc_metadata_installation_test.go b/tests/integration/helm_otc_metadata_installation_test.go index 762edd42b1..5e847b13ec 100644 --- a/tests/integration/helm_otc_metadata_installation_test.go +++ b/tests/integration/helm_otc_metadata_installation_test.go @@ -249,6 +249,7 @@ func Test_Helm_Default_OT_Metadata(t *testing.T) { sample := metricsSamples[0] labels := sample.Labels expectedLabels := receivermock.Labels{ + "cluster": "kubernetes", "_origin": "kubernetes", "container": "receiver-mock", "deployment": "receiver-mock", @@ -328,6 +329,7 @@ func Test_Helm_Default_OT_Metadata(t *testing.T) { Assess("expected container log metadata is present for log generator deployment", stepfuncs.WaitUntilExpectedLogsPresent( logsGeneratorCount, map[string]string{ + "cluster": "kubernetes", "_collector": "kubernetes", "namespace": internal.LogsGeneratorName, "pod_labels_app": internal.LogsGeneratorName, @@ -377,6 +379,7 @@ func Test_Helm_Default_OT_Metadata(t *testing.T) { Assess("logs from node systemd present", stepfuncs.WaitUntilExpectedLogsPresent( 10, // we don't really control this, just want to check if the logs show up map[string]string{ + "cluster": "kubernetes", "_sourceName": "", "_sourceCategory": "kubernetes/system", "_sourceHost": "", @@ -390,6 +393,7 @@ func Test_Helm_Default_OT_Metadata(t *testing.T) { Assess("logs from kubelet present", stepfuncs.WaitUntilExpectedLogsPresent( 1, // we don't really control this, just want to check if the logs show up map[string]string{ + "cluster": "kubernetes", "_sourceName": "k8s_kubelet", "_sourceCategory": "kubernetes/kubelet", "_sourceHost": "", diff --git a/tests/integration/helm_otelcol_logs_test.go b/tests/integration/helm_otelcol_logs_test.go index fa0ef7186f..18998f3c0d 100644 --- a/tests/integration/helm_otelcol_logs_test.go +++ b/tests/integration/helm_otelcol_logs_test.go @@ -124,6 +124,7 @@ func Test_Helm_Otelcol_Logs(t *testing.T) { logsGeneratorCount, map[string]string{ "_collector": "kubernetes", + "cluster": "kubernetes", "namespace": internal.LogsGeneratorName, "pod_labels_app": internal.LogsGeneratorName, "container": internal.LogsGeneratorName, diff --git a/tests/integration/values/values_helm_otelcol_logs.yaml b/tests/integration/values/values_helm_otelcol_logs.yaml index 9b49fad700..0d00069213 100644 --- a/tests/integration/values/values_helm_otelcol_logs.yaml +++ b/tests/integration/values/values_helm_otelcol_logs.yaml @@ -26,6 +26,7 @@ metadata: - memory_limiter - groupbyattrs/containers - k8s_tagger + - resource/add_cluster - source/containers - resource/containers_copy_node_to_host - batch