From 7742c04daf69ba442f2a07dde65f1360498aa0b2 Mon Sep 17 00:00:00 2001 From: Katarzyna Kujawa Date: Tue, 30 Apr 2024 12:34:37 +0200 Subject: [PATCH] feat: add support for namespace annotations --- .changelog/3684.added.txt | 1 + .../sumologic/conf/logs/otelcol/config.yaml | 6 ++ .../metadata_logs_otc/debug.output.yaml | 6 ++ .../debug_with_sumologic_mock.output.yaml | 6 ++ ...debug_with_sumologic_mock_http.output.yaml | 6 ++ .../metadata_logs_otc/otel.output.yaml | 6 ++ .../metadata_logs_otc/templates.output.yaml | 6 ++ tests/integration/features.go | 54 ++++++++++++---- .../helm_namespace_annotations_test.go | 20 ++++++ tests/integration/internal/constants.go | 3 + .../values_helm_namespace_annotations.yaml | 7 +++ .../yamls/namespace-annotations-test.yaml | 61 +++++++++++++++++++ 12 files changed, 170 insertions(+), 12 deletions(-) create mode 100644 .changelog/3684.added.txt create mode 100644 tests/integration/helm_namespace_annotations_test.go create mode 100644 tests/integration/values/values_helm_namespace_annotations.yaml create mode 100644 tests/integration/yamls/namespace-annotations-test.yaml diff --git a/.changelog/3684.added.txt b/.changelog/3684.added.txt new file mode 100644 index 0000000000..46b27ca66e --- /dev/null +++ b/.changelog/3684.added.txt @@ -0,0 +1 @@ +feat: add support for namespace annotations to set sourceCategory, sourceCategoryPrefix, sourceCategoryReplaceDash, sourceHost, sourceName \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml index dc44be4305..351d910819 100644 --- a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml @@ -280,6 +280,9 @@ processors: annotations: - key: "*" tag_name: "pod_annotations_%s" + namespace_annotations: + - key: "*" + tag_name: namespace_annotations_%s delimiter: "_" labels: - key: "*" @@ -342,6 +345,8 @@ processors: attributes: - action: delete pattern: ^pod_annotations_.* + - action: delete + pattern: ^namespace_annotations_.* {{ end }} {{ if .Values.sumologic.logs.container.enabled }} @@ -370,6 +375,7 @@ processors: {{ if .Values.sumologic.logs.container.enabled }} source/containers: annotation_prefix: "pod_annotations_" + namespace_annotation_prefix: "namespace_annotations_" collector: {{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }} container_annotations: enabled: {{ .Values.sumologic.logs.container.perContainerAnnotationsEnabled }} diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml index 424b3b292a..827a0903a7 100644 --- a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug.output.yaml @@ -171,6 +171,9 @@ data: - podName - serviceName - statefulSetName + namespace_annotations: + - key: '*' + tag_name: namespace_annotations_%s namespace_labels: - key: '*' tag_name: namespace_labels_%s @@ -196,6 +199,8 @@ data: attributes: - action: delete pattern: ^pod_annotations_.* + - action: delete + pattern: ^namespace_annotations_.* resource/remove_pod_name: attributes: - action: delete @@ -223,6 +228,7 @@ data: namespace: "" node: "" pod: "" + namespace_annotation_prefix: namespace_annotations_ pod_key: pod pod_name_key: pod_name pod_template_hash_key: pod_labels_pod-template-hash diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock.output.yaml index 81262a0fae..bf687c2c40 100644 --- a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock.output.yaml @@ -180,6 +180,9 @@ data: - podName - serviceName - statefulSetName + namespace_annotations: + - key: '*' + tag_name: namespace_annotations_%s namespace_labels: - key: '*' tag_name: namespace_labels_%s @@ -205,6 +208,8 @@ data: attributes: - action: delete pattern: ^pod_annotations_.* + - action: delete + pattern: ^namespace_annotations_.* resource/remove_pod_name: attributes: - action: delete @@ -232,6 +237,7 @@ data: namespace: "" node: "" pod: "" + namespace_annotation_prefix: namespace_annotations_ pod_key: pod pod_name_key: pod_name pod_template_hash_key: pod_labels_pod-template-hash diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock_http.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock_http.output.yaml index b914eeec80..d4a162115a 100644 --- a/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock_http.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_logs_otc/debug_with_sumologic_mock_http.output.yaml @@ -194,6 +194,9 @@ data: - podName - serviceName - statefulSetName + namespace_annotations: + - key: '*' + tag_name: namespace_annotations_%s namespace_labels: - key: '*' tag_name: namespace_labels_%s @@ -219,6 +222,8 @@ data: attributes: - action: delete pattern: ^pod_annotations_.* + - action: delete + pattern: ^namespace_annotations_.* resource/remove_pod_name: attributes: - action: delete @@ -246,6 +251,7 @@ data: namespace: "" node: "" pod: "" + namespace_annotation_prefix: namespace_annotations_ pod_key: pod pod_name_key: pod_name pod_template_hash_key: pod_labels_pod-template-hash diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml index 2c3e6db8d5..45734c54f7 100644 --- a/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml @@ -169,6 +169,9 @@ data: - podName - serviceName - statefulSetName + namespace_annotations: + - key: '*' + tag_name: namespace_annotations_%s namespace_labels: - key: '*' tag_name: namespace_labels_%s @@ -194,6 +197,8 @@ data: attributes: - action: delete pattern: ^pod_annotations_.* + - action: delete + pattern: ^namespace_annotations_.* resource/remove_pod_name: attributes: - action: delete @@ -221,6 +226,7 @@ data: namespace: "" node: "" pod: "" + namespace_annotation_prefix: namespace_annotations_ pod_key: pod pod_name_key: pod_name pod_template_hash_key: pod_labels_pod-template-hash diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml index f42c6555c4..dbaccf60ee 100644 --- a/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml @@ -169,6 +169,9 @@ data: - podName - serviceName - statefulSetName + namespace_annotations: + - key: '*' + tag_name: namespace_annotations_%s namespace_labels: - key: '*' tag_name: namespace_labels_%s @@ -209,6 +212,8 @@ data: attributes: - action: delete pattern: ^pod_annotations_.* + - action: delete + pattern: ^namespace_annotations_.* resource/remove-container: attributes: - action: delete @@ -248,6 +253,7 @@ data: namespace: my_containers_excludeNamespaceRegex node: my_containers_excludeHostRegex pod: my_containers_excludePodRegex + namespace_annotation_prefix: namespace_annotations_ pod_key: pod pod_name_key: pod_name pod_template_hash_key: pod_labels_pod-template-hash diff --git a/tests/integration/features.go b/tests/integration/features.go index 05de81a934..dd0af81603 100644 --- a/tests/integration/features.go +++ b/tests/integration/features.go @@ -37,18 +37,19 @@ const ( // number of log records in single loop with default multiline support only, see: tests/integration/yamls/pod_multiline_long_lines.yaml logRecords = 4 + 10 // number of log records in single loop with multiple multilines support, see: tests/integration/yamls/pod_multiline_long_lines.yaml - multipleLogRecords = 4 + 4 - logLoops = 500 // number of loops in which logs are generated, see: tests/integration/yamls/pod_multiline_long_lines.yaml - multilineLogCount uint = logRecords * logLoops - multipleMultilineLogCount uint = multipleLogRecords * logLoops - tracesPerExporter uint = 5 // number of traces generated per exporter - spansPerTrace uint = 2 - Prometheus MetricsCollector = "prometheus" - Otelcol MetricsCollector = "otelcol" - TailingSidecarCount uint = 150 // number of logs generated by tailing sidecar test (50 * 3), see: tests/inegration/yamls/tailing-sidecar-test.yaml - AnnotationsLogsCount uint = 150 // number of logs generated by containers used in annotations test (50 * 3), see: tests/inegration/yamls/annotations-test.yaml - curlAppMaxWaitTime uint = 180 - curlAppSleepInterval uint = 5 + multipleLogRecords = 4 + 4 + logLoops = 500 // number of loops in which logs are generated, see: tests/integration/yamls/pod_multiline_long_lines.yaml + multilineLogCount uint = logRecords * logLoops + multipleMultilineLogCount uint = multipleLogRecords * logLoops + tracesPerExporter uint = 5 // number of traces generated per exporter + spansPerTrace uint = 2 + Prometheus MetricsCollector = "prometheus" + Otelcol MetricsCollector = "otelcol" + TailingSidecarCount uint = 150 // number of logs generated by tailing sidecar test (50 * 3), see: tests/inegration/yamls/tailing-sidecar-test.yaml + AnnotationsLogsCount uint = 150 // number of logs generated by containers used in annotations test (50 * 3), see: tests/inegration/yamls/annotations-test.yaml + NamespaceAnnotationsLogsCount uint = 150 // number of logs generated by containers used in annotations test (50 * 3), see: tests/inegration/yamls/namespace-annotations-test.yaml + curlAppMaxWaitTime uint = 180 + curlAppSleepInterval uint = 5 ) func GetMetricsFeature(expectedMetrics []string, metricsCollector MetricsCollector) features.Feature { @@ -810,6 +811,35 @@ func GetAnnotationsFeature() features.Feature { Feature() } +func GetNamespaceAnnotationsFeature() features.Feature { + return features.New("namespace annotations test"). + Setup(stepfuncs.KubectlApplyFOpt(internal.NamespaceAnnotationsTest, internal.NamespaceAnnotationsTestNamespace)). + Assess("pod annotations are applied", stepfuncs.WaitUntilExpectedLogsPresent( + AnnotationsLogsCount, + map[string]string{ + "namespace": internal.NamespaceAnnotationsTestNamespace, + "_sourceCategory": "namespace#Source#Category#PrefixnamespaceSourceCategory", + "_sourceHost": "namespaceSourceHost", + "_sourceName": "namespaceSourceName", + }, + waitDuration, + tickDuration, + )). + Assess("container annotation is applied", stepfuncs.WaitUntilExpectedLogsPresent( + AnnotationsLogsCount, + map[string]string{ + "namespace": internal.NamespaceAnnotationsTestNamespace, + "_sourceCategory": "podSource!Category!PrefixpodSourceCategory", + "_sourceHost": "podSourceHost", + "_sourceName": "podSourceName", + }, + waitDuration, + tickDuration, + )). + Teardown(stepfuncs.KubectlDeleteFOpt(internal.NamespaceAnnotationsTest, internal.NamespaceAnnotationsTestNamespace)). + Feature() +} + type featureCheck func(*features.FeatureBuilder) *features.FeatureBuilder func GetInstallFeature(installChecks []featureCheck) features.Feature { diff --git a/tests/integration/helm_namespace_annotations_test.go b/tests/integration/helm_namespace_annotations_test.go new file mode 100644 index 0000000000..6e56b5607d --- /dev/null +++ b/tests/integration/helm_namespace_annotations_test.go @@ -0,0 +1,20 @@ +//go:build onlylatest +// +build onlylatest + +package integration + +import ( + "testing" +) + +func Test_Helm_Namespace_Annotations(t *testing.T) { + installChecks := []featureCheck{ + CheckOtelcolMetadataLogsInstall, + } + + featInstall := GetInstallFeature(installChecks) + + featAnnotationsTest := GetNamespaceAnnotationsFeature() + + testenv.Test(t, featInstall, featAnnotationsTest) +} diff --git a/tests/integration/internal/constants.go b/tests/integration/internal/constants.go index a7264d48d6..144594627e 100644 --- a/tests/integration/internal/constants.go +++ b/tests/integration/internal/constants.go @@ -60,6 +60,9 @@ const ( AnnotationsTestNamespace = "annotations-test" AnnotationsTest = "yamls/annotations-test.yaml" + NamespaceAnnotationsTestNamespace = "namespace-annotations-test" + NamespaceAnnotationsTest = "yamls/namespace-annotations-test.yaml" + NginxTelegrafMetricsTest = "yamls/nginx.yaml" NginxTelegrafNamespace = "nginx" diff --git a/tests/integration/values/values_helm_namespace_annotations.yaml b/tests/integration/values/values_helm_namespace_annotations.yaml new file mode 100644 index 0000000000..4a25fabfec --- /dev/null +++ b/tests/integration/values/values_helm_namespace_annotations.yaml @@ -0,0 +1,7 @@ +sumologic: + events: + enabled: false + metrics: + enabled: false + traces: + enabled: false diff --git a/tests/integration/yamls/namespace-annotations-test.yaml b/tests/integration/yamls/namespace-annotations-test.yaml new file mode 100644 index 0000000000..3e6239d719 --- /dev/null +++ b/tests/integration/yamls/namespace-annotations-test.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: + sumologic.com/include: "true" + sumologic.com/sourceCategory: "namespaceSourceCategory" + sumologic.com/sourceCategoryPrefix: "namespace-Source-Category-Prefix" + sumologic.com/sourceCategoryReplaceDash: "#" + sumologic.com/sourceHost: "namespaceSourceHost" + sumologic.com/sourceName: "namespaceSourceName" + name: namespace-annotations-test +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-a + namespace: namespace-annotations-test +spec: + containers: + - name: container1 + image: bash + args: + - /usr/local/bin/bash + - -c + - > + i=0; for i in {1..50}; do + echo "example0: $i $(date)" + echo "example1: $i $(date)" + echo "example2: $i $(date)" + sleep 1; + done; while true; do + sleep 1; + done; +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-b + namespace: namespace-annotations-test + annotations: + sumologic.com/sourceCategory: "podSourceCategory" + sumologic.com/sourceCategoryPrefix: "podSource-Category-Prefix" + sumologic.com/sourceCategoryReplaceDash: "!" + sumologic.com/sourceHost: "podSourceHost" + sumologic.com/sourceName: "podSourceName" +spec: + containers: + - name: container1 + image: bash + args: + - /usr/local/bin/bash + - -c + - > + i=0; for i in {1..50}; do + echo "example0: $i $(date)" + echo "example1: $i $(date)" + echo "example2: $i $(date)" + sleep 1; + done; while true; do + sleep 1; + done;