feat: add support for namespace annotations

SumoLogic · May 9, 2024 · 7742c04 · 7742c04
1 parent 129e1ce
commit 7742c04
Show file tree

Hide file tree

Showing 12 changed files with 170 additions and 12 deletions.
diff --git a/.changelog/3684.added.txt b/.changelog/3684.added.txt
@@ -0,0 +1 @@
+feat: add support for namespace annotations to set sourceCategory, sourceCategoryPrefix, sourceCategoryReplaceDash, sourceHost, sourceName
diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml
@@ -280,6 +280,9 @@ processors:
       annotations:
         - key: "*"
           tag_name: "pod_annotations_%s"
+      namespace_annotations:
+        - key: "*"
+          tag_name: namespace_annotations_%s
       delimiter: "_"
       labels:
         - key: "*"
@@ -342,6 +345,8 @@ processors:
     attributes:
       - action: delete
         pattern: ^pod_annotations_.*
+      - action: delete
+        pattern: ^namespace_annotations_.*
 {{ end }}
 
 {{ if .Values.sumologic.logs.container.enabled }}
@@ -370,6 +375,7 @@ processors:
 {{ if .Values.sumologic.logs.container.enabled }}
   source/containers:
     annotation_prefix: "pod_annotations_"
+    namespace_annotation_prefix: "namespace_annotations_"
     collector: {{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}
     container_annotations:
       enabled: {{ .Values.sumologic.logs.container.perContainerAnnotationsEnabled }}

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml
@@ -171,6 +171,9 @@ data:
           - podName
           - serviceName
           - statefulSetName
+          namespace_annotations:
+          - key: '*'
+            tag_name: namespace_annotations_%s
           namespace_labels:
           - key: '*'
             tag_name: namespace_labels_%s
@@ -196,6 +199,8 @@ data:
         attributes:
         - action: delete
           pattern: ^pod_annotations_.*
+        - action: delete
+          pattern: ^namespace_annotations_.*
       resource/remove_pod_name:
         attributes:
         - action: delete
@@ -223,6 +228,7 @@ data:
           namespace: ""
           node: ""
           pod: ""
+        namespace_annotation_prefix: namespace_annotations_
         pod_key: pod
         pod_name_key: pod_name
         pod_template_hash_key: pod_labels_pod-template-hash

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock.output.yaml
@@ -180,6 +180,9 @@ data:
           - podName
           - serviceName
           - statefulSetName
+          namespace_annotations:
+          - key: '*'
+            tag_name: namespace_annotations_%s
           namespace_labels:
           - key: '*'
             tag_name: namespace_labels_%s
@@ -205,6 +208,8 @@ data:
         attributes:
         - action: delete
           pattern: ^pod_annotations_.*
+        - action: delete
+          pattern: ^namespace_annotations_.*
       resource/remove_pod_name:
         attributes:
         - action: delete
@@ -232,6 +237,7 @@ data:
           namespace: ""
           node: ""
           pod: ""
+        namespace_annotation_prefix: namespace_annotations_
         pod_key: pod
         pod_name_key: pod_name
         pod_template_hash_key: pod_labels_pod-template-hash

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock_http.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock_http.output.yaml
@@ -194,6 +194,9 @@ data:
           - podName
           - serviceName
           - statefulSetName
+          namespace_annotations:
+          - key: '*'
+            tag_name: namespace_annotations_%s
           namespace_labels:
           - key: '*'
             tag_name: namespace_labels_%s
@@ -219,6 +222,8 @@ data:
         attributes:
         - action: delete
           pattern: ^pod_annotations_.*
+        - action: delete
+          pattern: ^namespace_annotations_.*
       resource/remove_pod_name:
         attributes:
         - action: delete
@@ -246,6 +251,7 @@ data:
           namespace: ""
           node: ""
           pod: ""
+        namespace_annotation_prefix: namespace_annotations_
         pod_key: pod
         pod_name_key: pod_name
         pod_template_hash_key: pod_labels_pod-template-hash

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml
@@ -169,6 +169,9 @@ data:
           - podName
           - serviceName
           - statefulSetName
+          namespace_annotations:
+          - key: '*'
+            tag_name: namespace_annotations_%s
           namespace_labels:
           - key: '*'
             tag_name: namespace_labels_%s
@@ -194,6 +197,8 @@ data:
         attributes:
         - action: delete
           pattern: ^pod_annotations_.*
+        - action: delete
+          pattern: ^namespace_annotations_.*
       resource/remove_pod_name:
         attributes:
         - action: delete
@@ -221,6 +226,7 @@ data:
           namespace: ""
           node: ""
           pod: ""
+        namespace_annotation_prefix: namespace_annotations_
         pod_key: pod
         pod_name_key: pod_name
         pod_template_hash_key: pod_labels_pod-template-hash

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml
@@ -169,6 +169,9 @@ data:
           - podName
           - serviceName
           - statefulSetName
+          namespace_annotations:
+          - key: '*'
+            tag_name: namespace_annotations_%s
           namespace_labels:
           - key: '*'
             tag_name: namespace_labels_%s
@@ -209,6 +212,8 @@ data:
         attributes:
         - action: delete
           pattern: ^pod_annotations_.*
+        - action: delete
+          pattern: ^namespace_annotations_.*
       resource/remove-container:
         attributes:
         - action: delete
@@ -248,6 +253,7 @@ data:
           namespace: my_containers_excludeNamespaceRegex
           node: my_containers_excludeHostRegex
           pod: my_containers_excludePodRegex
+        namespace_annotation_prefix: namespace_annotations_
         pod_key: pod
         pod_name_key: pod_name
         pod_template_hash_key: pod_labels_pod-template-hash

diff --git a/tests/integration/features.go b/tests/integration/features.go
@@ -37,18 +37,19 @@ const (
 	// number of log records in single loop with default multiline support only, see: tests/integration/yamls/pod_multiline_long_lines.yaml
 	logRecords = 4 + 10
 	// number of log records in single loop with multiple multilines support, see: tests/integration/yamls/pod_multiline_long_lines.yaml
-	multipleLogRecords                         = 4 + 4
-	logLoops                                   = 500 // number of loops in which logs are generated, see: tests/integration/yamls/pod_multiline_long_lines.yaml
-	multilineLogCount         uint             = logRecords * logLoops
-	multipleMultilineLogCount uint             = multipleLogRecords * logLoops
-	tracesPerExporter         uint             = 5 // number of traces generated per exporter
-	spansPerTrace             uint             = 2
-	Prometheus                MetricsCollector = "prometheus"
-	Otelcol                   MetricsCollector = "otelcol"
-	TailingSidecarCount       uint             = 150 // number of logs generated by tailing sidecar test (50 * 3), see: tests/inegration/yamls/tailing-sidecar-test.yaml
-	AnnotationsLogsCount      uint             = 150 // number of logs generated by containers used in annotations test (50 * 3), see: tests/inegration/yamls/annotations-test.yaml
-	curlAppMaxWaitTime        uint             = 180
-	curlAppSleepInterval      uint             = 5
+	multipleLogRecords                             = 4 + 4
+	logLoops                                       = 500 // number of loops in which logs are generated, see: tests/integration/yamls/pod_multiline_long_lines.yaml
+	multilineLogCount             uint             = logRecords * logLoops
+	multipleMultilineLogCount     uint             = multipleLogRecords * logLoops
+	tracesPerExporter             uint             = 5 // number of traces generated per exporter
+	spansPerTrace                 uint             = 2
+	Prometheus                    MetricsCollector = "prometheus"
+	Otelcol                       MetricsCollector = "otelcol"
+	TailingSidecarCount           uint             = 150 // number of logs generated by tailing sidecar test (50 * 3), see: tests/inegration/yamls/tailing-sidecar-test.yaml
+	AnnotationsLogsCount          uint             = 150 // number of logs generated by containers used in annotations test (50 * 3), see: tests/inegration/yamls/annotations-test.yaml
+	NamespaceAnnotationsLogsCount uint             = 150 // number of logs generated by containers used in annotations test (50 * 3), see: tests/inegration/yamls/namespace-annotations-test.yaml
+	curlAppMaxWaitTime            uint             = 180
+	curlAppSleepInterval          uint             = 5
 )
 
 func GetMetricsFeature(expectedMetrics []string, metricsCollector MetricsCollector) features.Feature {
@@ -810,6 +811,35 @@ func GetAnnotationsFeature() features.Feature {
 		Feature()
 }
 
+func GetNamespaceAnnotationsFeature() features.Feature {
+	return features.New("namespace annotations test").
+		Setup(stepfuncs.KubectlApplyFOpt(internal.NamespaceAnnotationsTest, internal.NamespaceAnnotationsTestNamespace)).
+		Assess("pod annotations are applied", stepfuncs.WaitUntilExpectedLogsPresent(
+			AnnotationsLogsCount,
+			map[string]string{
+				"namespace":       internal.NamespaceAnnotationsTestNamespace,
+				"_sourceCategory": "namespace#Source#Category#PrefixnamespaceSourceCategory",
+				"_sourceHost":     "namespaceSourceHost",
+				"_sourceName":     "namespaceSourceName",
+			},
+			waitDuration,
+			tickDuration,
+		)).
+		Assess("container annotation is applied", stepfuncs.WaitUntilExpectedLogsPresent(
+			AnnotationsLogsCount,
+			map[string]string{
+				"namespace":       internal.NamespaceAnnotationsTestNamespace,
+				"_sourceCategory": "podSource!Category!PrefixpodSourceCategory",
+				"_sourceHost":     "podSourceHost",
+				"_sourceName":     "podSourceName",
+			},
+			waitDuration,
+			tickDuration,
+		)).
+		Teardown(stepfuncs.KubectlDeleteFOpt(internal.NamespaceAnnotationsTest, internal.NamespaceAnnotationsTestNamespace)).
+		Feature()
+}
+
 type featureCheck func(*features.FeatureBuilder) *features.FeatureBuilder
 
 func GetInstallFeature(installChecks []featureCheck) features.Feature {

diff --git a/tests/integration/helm_namespace_annotations_test.go b/tests/integration/helm_namespace_annotations_test.go
@@ -0,0 +1,20 @@
+//go:build onlylatest
+// +build onlylatest
+
+package integration
+
+import (
+	"testing"
+)
+
+func Test_Helm_Namespace_Annotations(t *testing.T) {
+	installChecks := []featureCheck{
+		CheckOtelcolMetadataLogsInstall,
+	}
+
+	featInstall := GetInstallFeature(installChecks)
+
+	featAnnotationsTest := GetNamespaceAnnotationsFeature()
+
+	testenv.Test(t, featInstall, featAnnotationsTest)
+}
diff --git a/tests/integration/internal/constants.go b/tests/integration/internal/constants.go
@@ -60,6 +60,9 @@ const (
 	AnnotationsTestNamespace = "annotations-test"
 	AnnotationsTest          = "yamls/annotations-test.yaml"
 
+	NamespaceAnnotationsTestNamespace = "namespace-annotations-test"
+	NamespaceAnnotationsTest          = "yamls/namespace-annotations-test.yaml"
+
 	NginxTelegrafMetricsTest = "yamls/nginx.yaml"
 	NginxTelegrafNamespace   = "nginx"
 

diff --git a/tests/integration/values/values_helm_namespace_annotations.yaml b/tests/integration/values/values_helm_namespace_annotations.yaml
@@ -0,0 +1,7 @@
+sumologic:
+  events:
+    enabled: false
+  metrics:
+    enabled: false
+  traces:
+    enabled: false
diff --git a/tests/integration/yamls/namespace-annotations-test.yaml b/tests/integration/yamls/namespace-annotations-test.yaml
@@ -0,0 +1,61 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations:
+    sumologic.com/include: "true"
+    sumologic.com/sourceCategory: "namespaceSourceCategory"
+    sumologic.com/sourceCategoryPrefix: "namespace-Source-Category-Prefix"
+    sumologic.com/sourceCategoryReplaceDash: "#"
+    sumologic.com/sourceHost: "namespaceSourceHost"
+    sumologic.com/sourceName: "namespaceSourceName"
+  name: namespace-annotations-test
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-a
+  namespace: namespace-annotations-test
+spec:
+  containers:
+    - name: container1
+      image: bash
+      args:
+        - /usr/local/bin/bash
+        - -c
+        - >
+          i=0; for i in {1..50}; do
+            echo "example0: $i $(date)"
+            echo "example1: $i $(date)"
+            echo "example2: $i $(date)"
+            sleep 1;
+          done; while true; do
+            sleep 1;
+          done;
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-b
+  namespace: namespace-annotations-test
+  annotations:
+    sumologic.com/sourceCategory: "podSourceCategory"
+    sumologic.com/sourceCategoryPrefix: "podSource-Category-Prefix"
+    sumologic.com/sourceCategoryReplaceDash: "!"
+    sumologic.com/sourceHost: "podSourceHost"
+    sumologic.com/sourceName: "podSourceName"
+spec:
+  containers:
+    - name: container1
+      image: bash
+      args:
+        - /usr/local/bin/bash
+        - -c
+        - >
+          i=0; for i in {1..50}; do
+            echo "example0: $i $(date)"
+            echo "example1: $i $(date)"
+            echo "example2: $i $(date)"
+            sleep 1;
+          done; while true; do
+            sleep 1;
+          done;