fix(logs): make built-in metadata consistent between fluentd and otel

CHANGELOG.md

@@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - fix(logs): make `excludeHostRegex` consistent between Otelcol and Fluentd [#2771]
   - The `sumologic.logs.container.excludeHostRegex` should filter on the Kubernetes node name,
     to be consistent with Fluentd and chart v2.
+- fix(logs): make built-in metadata consistent between fluentd and otel [#2801]
 
 [#2724]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2724
 [#2745]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2745
@@ -50,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#2791]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2791
 [#2773]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2773
 [#2790]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2790
+[#2801]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2801
 [v1.15.3-sumo-0]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/releases/tag/v1.15.3-sumo-0
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v3.0.0-beta.0...main
 

deploy/helm/sumologic/conf/logs/otelcol/config.yaml

@@ -147,10 +147,23 @@ processors:
     attributes:
       - pattern: ^pod_annotations_.*
         action: delete
-  resource/remove_k8s_pod_pod_name:
+  resource/remove_pod_name:
     attributes:
       - action: delete
-        key: k8s.pod.pod_name
+        key: pod_name
+  # As sourceprocessor can't set these attributes to be empty, we do it here instead
+  # If they're defined to be anything else in sourceprocessor, those values will overwrite these
+  resource/set_empty_source_metadata:
+    attributes:
+    - key: _sourceCategory
+      action: insert
+      value: ""
+    - key: _sourceHost
+      action: insert
+      value: ""
+    - key: _sourceName
+      action: insert
+      value: ""
   groupbyattrs/containers:
     keys:
       - k8s.container.id
@@ -200,8 +213,8 @@ processors:
       k8s.node.name: {{ .Values.sumologic.logs.container.excludeHostRegex | quote }}
     annotation_prefix: "pod_annotations_"
     pod_template_hash_key: "pod_labels_pod-template-hash"
-    pod_name_key: "k8s.pod.pod_name"
-    pod_key: "k8s.pod.name"
+    pod_name_key: "pod_name"
+    pod_key: "pod"
     container_annotations:
       enabled: {{ .Values.sumologic.logs.container.perContainerAnnotationsEnabled }}
       prefixes: {{ toJson .Values.sumologic.logs.container.perContainerAnnotationPrefixes }}
@@ -366,17 +379,18 @@ service:
         - groupbyattrs/containers
         - k8s_tagger
         - resource/add_cluster
+        - resource/set_empty_source_metadata
+        - resource/containers_copy_node_to_host
+        - sumologic_schema
         - source/containers
 {{- if .Values.sumologic.logs.container.otelcol.extraProcessors }}
 {{- range $processor := .Values.sumologic.logs.container.otelcol.extraProcessors }}
 {{ printf "- %s" ( $processor | keys | first ) | indent 8 }}
 {{- end }}
 {{- end }}
-        - resource/remove_k8s_pod_pod_name
+        - resource/remove_pod_name
         - resource/drop_annotations
         - attributes/remove_fluent_tag
-        - resource/containers_copy_node_to_host
-        - sumologic_schema
         - batch
       exporters:
         - sumologic/containers
@@ -438,17 +452,18 @@ service:
         - groupbyattrs/containers
         - k8s_tagger
         - resource/add_cluster
+        - resource/set_empty_source_metadata
+        - resource/containers_copy_node_to_host
+        - sumologic_schema
         - source/containers
         - logstransform/containers_parse_json
 {{- if .Values.sumologic.logs.container.otelcol.extraProcessors }}
 {{- range $processor := .Values.sumologic.logs.container.otelcol.extraProcessors }}
 {{ printf "- %s" ( $processor | keys | first ) | indent 8 }}
 {{- end }}
 {{- end }}
-        - resource/remove_k8s_pod_pod_name
+        - resource/remove_pod_name
         - resource/drop_annotations
-        - resource/containers_copy_node_to_host
-        - sumologic_schema
         - batch
       exporters:
         - sumologic/containers

deploy/helm/sumologic/values.yaml

@@ -258,11 +258,11 @@ sumologic:
         extraProcessors: []
 
       ## Set the _sourceHost metadata field in Sumo Logic.
-      sourceHost: "%{k8s.pod.hostname}"
+      sourceHost: ""
       ## Set the _sourceName metadata field in Sumo Logic.
-      sourceName: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}"
+      sourceName: "%{namespace}.%{pod}.%{container}"
       ## Set the _sourceCategory metadata field in Sumo Logic.
-      sourceCategory: "%{k8s.namespace.name}/%{k8s.pod.pod_name}"
+      sourceCategory: "%{namespace}/%{pod_name}"
       ## Set the prefix, for _sourceCategory metadata.
       sourceCategoryPrefix: "kubernetes/"
       ## Used to replace - with another character.

docs/collecting-container-logs.md

@@ -66,22 +66,33 @@ sumologic:
   logs:
     container:
       ## Set the _sourceHost metadata field in Sumo Logic.
-      sourceHost: "%{k8s.pod.hostname}"
+      sourceHost: ""
       ## Set the _sourceName metadata field in Sumo Logic.
-      sourceName: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}"
+      sourceName: "%{namespace}.%{pod}.%{container}"
       ## Set the _sourceCategory metadata field in Sumo Logic.
-      sourceCategory: "%{k8s.namespace.name}/%{k8s.pod.pod_name}"
+      sourceCategory: "%{namespace}/%{pod_name}"
       ## Set the prefix, for _sourceCategory metadata.
       sourceCategoryPrefix: "kubernetes/"
       ## Used to replace - with another character.
       sourceCategoryReplaceDash: "/"
 ```
 
 As can be seen in the above example, these fields can contain templates of the form `%{field_name}`, where `field_name` is the name
-of a resource attribute. Available resource attributes include [OpenTelemetry Kubernetes resource attributes][opentelemetry_k8s],
-in addition to the following:
+of a resource attribute. Available resource attributes include the values of `sumologic.logs.fields`, which by default are:
 
 - `cluster`
+- `container`
+- `daemonset`
+- `deployment`
+- `host`
+- `namespace`
+- `node`
+- `pod`
+- `service`
+- `statefulset`
+
+in addition to the following:
+
 - `_collector`
 - `pod_labels_*` where `*` is the Pod label name
 
@@ -265,7 +276,6 @@ sumologic:
 [configuration]: https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/configuration.md
 [values]: /deploy/helm/sumologic/values.yaml
 [source_name]: https://help.sumologic.com/docs/send-data/reference-information/metadata-naming-conventions/#Source_Name
-[opentelemetry_k8s]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/k8s.md
 [filter_processor_docs]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.69.0/processor/filterprocessor/README.md
 [opentelemetry_processors]: https://opentelemetry.io/docs/collector/configuration/#processors
 [attributes_processor_docs]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.69.0/processor/attributesprocessor/README.md

tests/helm/metadata_logs_otc/static/otel.output.yaml

@@ -246,10 +246,21 @@ data:
         attributes:
         - action: delete
           pattern: ^pod_annotations_.*
-      resource/remove_k8s_pod_pod_name:
+      resource/remove_pod_name:
         attributes:
         - action: delete
-          key: k8s.pod.pod_name
+          key: pod_name
+      resource/set_empty_source_metadata:
+        attributes:
+        - action: insert
+          key: _sourceCategory
+          value: ""
+        - action: insert
+          key: _sourceHost
+          value: ""
+        - action: insert
+          key: _sourceName
+          value: ""
       source/containers:
         annotation_prefix: pod_annotations_
         collector: kubernetes
@@ -261,14 +272,14 @@ data:
           k8s.namespace.name: ""
           k8s.node.name: ""
           k8s.pod.name: ""
-        pod_key: k8s.pod.name
-        pod_name_key: k8s.pod.pod_name
+        pod_key: pod
+        pod_name_key: pod_name
         pod_template_hash_key: pod_labels_pod-template-hash
-        source_category: '%{k8s.namespace.name}/%{k8s.pod.pod_name}'
+        source_category: '%{namespace}/%{pod_name}'
         source_category_prefix: kubernetes/
         source_category_replace_dash: /
-        source_host: '%{k8s.pod.hostname}'
-        source_name: '%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}'
+        source_host: ""
+        source_name: '%{namespace}.%{pod}.%{container}'
       source/kubelet:
         collector: kubernetes
         source_category: kubelet
@@ -310,12 +321,13 @@ data:
           - groupbyattrs/containers
           - k8s_tagger
           - resource/add_cluster
+          - resource/set_empty_source_metadata
+          - resource/containers_copy_node_to_host
+          - sumologic_schema
           - source/containers
           - logstransform/containers_parse_json
-          - resource/remove_k8s_pod_pod_name
+          - resource/remove_pod_name
           - resource/drop_annotations
-          - resource/containers_copy_node_to_host
-          - sumologic_schema
           - batch
           receivers:
           - otlp

tests/helm/metadata_logs_otc/static/templates.output.yaml

@@ -246,10 +246,21 @@ data:
         attributes:
         - action: delete
           pattern: ^pod_annotations_.*
-      resource/remove_k8s_pod_pod_name:
+      resource/remove_pod_name:
         attributes:
         - action: delete
-          key: k8s.pod.pod_name
+          key: pod_name
+      resource/set_empty_source_metadata:
+        attributes:
+        - action: insert
+          key: _sourceCategory
+          value: ""
+        - action: insert
+          key: _sourceHost
+          value: ""
+        - action: insert
+          key: _sourceName
+          value: ""
       source/containers:
         annotation_prefix: pod_annotations_
         collector: my_collectorName
@@ -261,8 +272,8 @@ data:
           k8s.namespace.name: my_containers_excludeNamespaceRegex
           k8s.node.name: my_containers_excludeHostRegex
           k8s.pod.name: my_containers_excludePodRegex
-        pod_key: k8s.pod.name
-        pod_name_key: k8s.pod.pod_name
+        pod_key: pod
+        pod_name_key: pod_name
         pod_template_hash_key: pod_labels_pod-template-hash
         source_category: my_containers_sourceCategory
         source_category_prefix: my_containers_sourceCategoryPrefix
@@ -310,12 +321,13 @@ data:
           - groupbyattrs/containers
           - k8s_tagger
           - resource/add_cluster
+          - resource/set_empty_source_metadata
+          - resource/containers_copy_node_to_host
+          - sumologic_schema
           - source/containers
           - logstransform/containers_parse_json
-          - resource/remove_k8s_pod_pod_name
+          - resource/remove_pod_name
           - resource/drop_annotations
-          - resource/containers_copy_node_to_host
-          - sumologic_schema
           - batch
           receivers:
           - otlp

tests/integration/helm_fluentbit_fluentd_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strings"
 	"testing"
 	"time"
 
@@ -310,10 +311,20 @@ func Test_Helm_FluentBit_Fluentd(t *testing.T) {
 				"host":           internal.NodeNameRegex,
 				"master_url":     ".+",
 				"node":           internal.NodeNameRegex,
-				// The values below are a result of a bug and should be fixed shortly, see https://github.com/SumoLogic/sumologic-kubernetes-collection/issues/2767
-				"_sourceName":     "undefined.undefined.undefined",
-				"_sourceCategory": "kubernetes/undefined/undefined",
-				"_sourceHost":     "undefined",
+				"_sourceName": fmt.Sprintf(
+					"%s\\.%s%s\\.%s",
+					internal.LogsGeneratorNamespace,
+					internal.LogsGeneratorName,
+					internal.PodDeploymentSuffixRegex,
+					internal.LogsGeneratorName,
+				),
+				"_sourceCategory": fmt.Sprintf(
+					"%s/%s/%s", // dashes instead of hyphens due to sourceCategoryReplaceDash
+					internal.ClusterName,
+					strings.ReplaceAll(internal.LogsGeneratorNamespace, "-", "/"),
+					strings.ReplaceAll(internal.LogsGeneratorName, "-", "/"), // this is the pod name prefix, in this case the Deployment name
+				),
+				"_sourceHost": internal.EmptyRegex,
 			},
 			internal.ReceiverMockNamespace,
 			internal.ReceiverMockServiceName,

tests/integration/helm_ot_default_test.go

@@ -399,7 +399,7 @@ func Test_Helm_Default_OT(t *testing.T) {
 					strings.ReplaceAll(internal.LogsGeneratorNamespace, "-", "/"),
 					strings.ReplaceAll(internal.LogsGeneratorName, "-", "/"), // this is the pod name prefix, in this case the deployment name
 				),
-				"_sourceHost": fmt.Sprintf("%s%s", internal.LogsGeneratorName, internal.PodDeploymentSuffixRegex),
+				"_sourceHost": internal.EmptyRegex,
 			},
 			internal.ReceiverMockNamespace,
 			internal.ReceiverMockServiceName,
@@ -431,7 +431,7 @@ func Test_Helm_Default_OT(t *testing.T) {
 					strings.ReplaceAll(internal.LogsGeneratorNamespace, "-", "/"),
 					strings.ReplaceAll(internal.LogsGeneratorName, "-", "/"), // this is the pod name prefix, in this case the DaemonSet name
 				),
-				"_sourceHost": fmt.Sprintf("%s%s", internal.LogsGeneratorName, internal.PodDaemonSetSuffixRegex),
+				"_sourceHost": internal.EmptyRegex,
 			},
 			internal.ReceiverMockNamespace,
 			internal.ReceiverMockServiceName,

tests/integration/helm_ot_fluentbit_logs_test.go

@@ -165,7 +165,7 @@ func Test_Helm_OT_FluentBit_Logs(t *testing.T) {
 					strings.ReplaceAll(internal.LogsGeneratorNamespace, "-", "/"),
 					strings.ReplaceAll(internal.LogsGeneratorName, "-", "/"), // this is the pod name prefix, in this case the deployment name
 				),
-				"_sourceHost": fmt.Sprintf("%s%s", internal.LogsGeneratorName, internal.PodDeploymentSuffixRegex),
+				"_sourceHost": internal.EmptyRegex,
 			},
 			internal.ReceiverMockNamespace,
 			internal.ReceiverMockServiceName,
@@ -197,7 +197,7 @@ func Test_Helm_OT_FluentBit_Logs(t *testing.T) {
 					strings.ReplaceAll(internal.LogsGeneratorNamespace, "-", "/"),
 					strings.ReplaceAll(internal.LogsGeneratorName, "-", "/"), // this is the pod name prefix, in this case the DaemonSet name
 				),
-				"_sourceHost": fmt.Sprintf("%s%s", internal.LogsGeneratorName, internal.PodDaemonSetSuffixRegex),
+				"_sourceHost": internal.EmptyRegex,
 			},
 			internal.ReceiverMockNamespace,
 			internal.ReceiverMockServiceName,

tests/integration/helm_otc_fips_metadata_installation_test.go

@@ -352,7 +352,7 @@ func Test_Helm_Default_OT_FIPS_Metadata(t *testing.T) {
 					strings.ReplaceAll(internal.LogsGeneratorNamespace, "-", "/"),
 					strings.ReplaceAll(internal.LogsGeneratorName, "-", "/"), // this is the pod name prefix, in this case the deployment name
 				),
-				"_sourceHost": fmt.Sprintf("%s%s", internal.LogsGeneratorName, internal.PodDeploymentSuffixRegex),
+				"_sourceHost": internal.EmptyRegex,
 			},
 			internal.ReceiverMockNamespace,
 			internal.ReceiverMockServiceName,
@@ -384,7 +384,7 @@ func Test_Helm_Default_OT_FIPS_Metadata(t *testing.T) {
 					strings.ReplaceAll(internal.LogsGeneratorNamespace, "-", "/"),
 					strings.ReplaceAll(internal.LogsGeneratorName, "-", "/"), // this is the pod name prefix, in this case the DaemonSet name
 				),
-				"_sourceHost": fmt.Sprintf("%s%s", internal.LogsGeneratorName, internal.PodDaemonSetSuffixRegex),
+				"_sourceHost": internal.EmptyRegex,
 			},
 			internal.ReceiverMockNamespace,
 			internal.ReceiverMockServiceName,

tests/integration/internal/constants.go

@@ -40,6 +40,7 @@ const (
 	PodDaemonSetSuffixRegex  = "-[a-z0-9]{4,5}"
 	NodeNameRegex            = ".*-control-plane" // node name for KinD TODO: get this from the cluster directly instead
 	NotUndefinedRegex        = "(?!undefined$).*"
+	EmptyRegex               = "^$"
 )
 
 // metrics we expect the receiver to get