From 8878ecc74a996b6d1dc18599a99f95953b906824 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Thu, 28 Apr 2022 08:46:48 +0200 Subject: [PATCH 1/9] feat(metadata/otc): upgrade to v0.49.0-sumo-0 Signed-off-by: Dominik Rosiek --- deploy/docs/Best_Practices.md | 5 +- deploy/docs/fluentd_otc_comparison.md | 44 +++++++-------- deploy/helm/sumologic/values.yaml | 54 ++----------------- .../static/basic.output.yaml | 13 +---- .../static/templates.output.yaml | 13 +---- .../static/basic.output.yaml | 2 +- .../static/basic.output.yaml | 2 +- .../collection-monitoring-false.output.yaml | 3 -- tests/helm/tracing/static/simple.output.yaml | 3 -- vagrant/values.yaml | 3 -- 10 files changed, 33 insertions(+), 109 deletions(-) diff --git a/deploy/docs/Best_Practices.md b/deploy/docs/Best_Practices.md index 8b97732ef0..e590b7036e 100644 --- a/deploy/docs/Best_Practices.md +++ b/deploy/docs/Best_Practices.md @@ -214,9 +214,6 @@ metadata: - action: extract key: fluent.tag pattern: ^containers\.var\.log\.pods\.(?P[^_]+)_(?P[^_]+)_(?P[a-f0-9\-]{36})\.(?P[^\._]+)\.(?P\d+)\.log$ - - action: insert - key: k8s.pod.uid - from_attribute: k8s_uid - action: delete key: k8s_uid - action: delete @@ -1058,7 +1055,7 @@ For [sumologic exporter][sumologic_exporter]: **because once filled in PVC never reduces its fill.** [batch_processor]: https://github.com/open-telemetry/opentelemetry-collector/tree/v0.47.0/processor/batchprocessor#batch-processor -[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/exporter/sumologicexporter#sumo-logic-exporter +[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/exporter/sumologicexporter#sumo-logic-exporter [filling_up_alert]: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup/ ### Compaction diff --git a/deploy/docs/fluentd_otc_comparison.md b/deploy/docs/fluentd_otc_comparison.md index 8740c2029d..f91aa3e3a5 100644 --- a/deploy/docs/fluentd_otc_comparison.md +++ b/deploy/docs/fluentd_otc_comparison.md @@ -40,8 +40,8 @@ | `disable_cookies` | Cookies are not used in Opentelemetry Collector | | `compress` | [exporters.sumologic.compress_encoding][otelcol_sumologic_config] set to `""` | | `compress_encoding` | [exporters.sumologic.compress_encoding][otelcol_sumologic_config] | -| `custom_fields` | [Resource processor][resource_processor] combined with [exporters.sumologic.metadata_attributes][otelcol_sumologic_config] | -| `custom_dimensions` | [Resource processor][resource_processor] combined with [exporters.sumologic.metadata_attributes][otelcol_sumologic_config] | +| `custom_fields` | [Resource processor][resource_processor] | +| `custom_dimensions` | [Resource processor][resource_processor] | Additional behavior: @@ -50,14 +50,14 @@ Additional behavior: | [record[_sumo_metadata][source_name]][source_name_precedence] taking precedence over `source_name` | Can be achieved by separate pipelines | | [record[_sumo_metadata][source_host]][source_host_precedence] taking precedence over `source_host` | Can be achieved by separate pipelines | | [record[_sumo_metadata][source_category]][source_category_precedence] taking precedence over `source_category` | Can be achieved by separate pipelines | -| [record[_sumo_metadata][fields]][fields_base] being base for fields | Can be achieved using [resource processor][resource_processor], separate pipelines and [exporter.sumologic.metadata_attributes][otelcol_sumologic_config] | +| [record[_sumo_metadata][fields]][fields_base] being base for fields | Can be achieved using [resource processor][resource_processor] and separate pipelines | [fields_base]: https://github.com/SumoLogic/fluentd-output-sumologic/blob/1.7.2/lib/fluent/plugin/out_sumologic.rb#L284-L285 [fluentd_output_plugin]: https://github.com/sumologic/fluentd-output-sumologic/tree/1.7.2#configuration -[otelcol_basic_confg]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/docs/Configuration.md#basic-configuration -[otelcol_proxy]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/docs/Configuration.md#proxy-support -[otelcol_source_config]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#config -[otelcol_sumologic_config]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter +[otelcol_basic_confg]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/docs/Configuration.md#basic-configuration +[otelcol_proxy]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/docs/Configuration.md#proxy-support +[otelcol_source_config]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#config +[otelcol_sumologic_config]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter [otelocl_tls_config]: https://github.com/open-telemetry/opentelemetry-collector/blob/v0.47.0/config/configtls/README.md#tls--mtls-configuration [resource_processor]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.47.0/processor/resourceprocessor#resource-processor [source_category_precedence]: https://github.com/SumoLogic/fluentd-output-sumologic/blob/1.7.2/lib/fluent/plugin/out_sumologic.rb#L278-L279 @@ -70,7 +70,7 @@ In order to receive prometheus data and for their initial processing [telegrafre It should cover [fluent-plugin-datapoint][fluent_plugin_datapoint] functionality and more. [fluent_plugin_datapoint]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-datapoint -[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/receiver/telegrafreceiver +[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/receiver/telegrafreceiver ### fluent-plugin-protobuf @@ -78,7 +78,7 @@ In order to receive prometheus data and for their initial processing [telegrafre It should cover [fluent_plugin_protobuf][fluent_plugin_protobuf] functionality and more. [fluent_plugin_protobuf]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-protobuf -[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/receiver/telegrafreceiver +[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/receiver/telegrafreceiver ### fluent-plugin-prometheus-format @@ -162,17 +162,17 @@ Sanitized pod name is name portion of the pod. Please consider following example [kube_daemonset]: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/ [kube_deployment]: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ [kube_statefulset]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset -[otelcol_annotations]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#pod-annotations +[otelcol_annotations]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#pod-annotations [otelcol_annotations_exclude]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/blob/v1.12.2-sumo-4/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb#L171-L183 -[otelcol_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/exporter/sumologicexporter#source-templates -[otelcol_undefined]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/processor/sourceprocessor/attribute_filler.go#L113-L123 -[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations -[source_filtering]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#filtering-section -[source_keys]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#keys-section -[source_processor]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#config -[source_processor_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#name-translation-and-template-keys +[otelcol_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/exporter/sumologicexporter#source-templates +[otelcol_undefined]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/processor/sourceprocessor/attribute_filler.go#L113-L123 +[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations +[source_filtering]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#filtering-section +[source_keys]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#keys-section +[source_processor]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#config +[source_processor_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#name-translation-and-template-keys [sumo_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-kubernetes-sumologic#fluent-plugin-kubernetes-sumologic -[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter +[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter ### fluent-plugin-kubernetes-metadata-filter @@ -190,8 +190,8 @@ Sanitized pod name is name portion of the pod. Please consider following example | `cache_ttl` | N/A | [fluent_plugin_k8s_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-kubernetes-metadata-filter#configuration -[k8sprocessor]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/processor/k8sprocessor -[k8sprocessor_field_extract]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/k8sprocessor#field-extract-config +[k8sprocessor]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/processor/k8sprocessor +[k8sprocessor_field_extract]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/k8sprocessor#field-extract-config [kubeconfig_auth_type]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.47.0/internal/k8sconfig/config.go#L53-L60 ### fluent-plugin-enhance-k8s-metadata @@ -209,7 +209,7 @@ Sanitized pod name is name portion of the pod. Please consider following example | `data_type` | N/A | [fluent_plugin_enhance_k8s_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-enhance-k8s-metadata#configuration -[pod_association]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/processor/k8sprocessor/doc.go#L17-L46 +[pod_association]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/processor/k8sprocessor/doc.go#L17-L46 ### fluent-plugin-events @@ -358,7 +358,7 @@ Events are not supported by `Opentelemetry Collector` [filter_processor]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.47.0/processor/filterprocessor#filter-processor [readme]: ../helm/sumologic/README.md -[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations +[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations [persistent_queue]: https://github.com/open-telemetry/opentelemetry-collector/tree/release/v0.37.x/exporter/exporterhelper#persistent-queue [file_storage_extension]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.37.x/extension/storage/filestorage diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 0ef84bcad5..65d3e5c623 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3543,10 +3543,6 @@ otelcol: source_name: "exporters.sumologic.source_name.replace" ## Desired host name. Useful if you want to override the source host configured for the source. source_host: "%{host.name}" - ## List of regexes for attributes which should be send as metadata - metadata_attributes: - - k8s.* - - host.name ## Timeout for every attempt to send data to Sumo Logic backend. Maximum connection timeout is 55s. timeout: 5s retry_on_failure: @@ -3585,7 +3581,7 @@ metadata: ## Configure image for Opentelemetry Collector (for logs and metrics) image: repository: public.ecr.aws/sumologic/sumologic-otel-collector - tag: 0.47.0-sumo-0 + tag: 0.49.0-sumo-0 pullPolicy: IfNotPresent securityContext: @@ -3693,10 +3689,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* routing_atttribute_to_drop: http_listener_v2_path sumologic/apiserver: metric_format: prometheus @@ -3709,10 +3701,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* routing_atttribute_to_drop: http_listener_v2_path sumologic/control_plane: metric_format: prometheus @@ -3725,10 +3713,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* routing_atttribute_to_drop: http_listener_v2_path sumologic/controller: metric_format: prometheus @@ -3741,10 +3725,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* routing_atttribute_to_drop: http_listener_v2_path sumologic/kubelet: metric_format: prometheus @@ -3757,10 +3737,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* routing_atttribute_to_drop: http_listener_v2_path sumologic/node: metric_format: prometheus @@ -3773,10 +3749,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* routing_atttribute_to_drop: http_listener_v2_path sumologic/scheduler: metric_format: prometheus @@ -3789,10 +3761,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* routing_atttribute_to_drop: http_listener_v2_path sumologic/state: metric_format: prometheus @@ -3805,10 +3773,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* routing_atttribute_to_drop: http_listener_v2_path processors: ## Configuration for Metrics Transform Processor @@ -4105,12 +4069,6 @@ metadata: sending_queue: enabled: true persistent_storage_enabled: '{{ .Values.metadata.persistence.enabled }}' - metadata_attributes: - ## Attributes to be send as fields - - _collector - - k8s.* - - pod_labels_.* - - namespace_labels_.* sumologic/systemd: log_format: json json_logs: @@ -4125,8 +4083,6 @@ metadata: sending_queue: enabled: true persistent_storage_enabled: '{{ .Values.metadata.persistence.enabled }}' - metadata_attributes: - - _collector processors: ## Common processors @@ -4198,6 +4154,7 @@ metadata: - k8s.container.name - k8s.namespace.name - k8s.pod.name + - _collector k8s_tagger: ## Has to be false to enrich metadata passthrough: false @@ -4279,11 +4236,8 @@ metadata: value: kubelet.service groupbyattrs/systemd: keys: - - _SYSTEMD_UNIT - - _HOSTNAME - - SYSLOG_FACILITY - - PRIORITY - _sourceName + - _collector source/systemd: collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' source_host: "%{_HOSTNAME}" @@ -4484,7 +4438,7 @@ otelgateway: podAnnotations: {} image: repository: public.ecr.aws/sumologic/sumologic-otel-collector - tag: 0.47.0-sumo-0 + tag: 0.49.0-sumo-0 pullPolicy: IfNotPresent livenessProbe: periodSeconds: 15 diff --git a/tests/helm/metadata_logs_otc/static/basic.output.yaml b/tests/helm/metadata_logs_otc/static/basic.output.yaml index ea4d56f742..47af0bb0b0 100644 --- a/tests/helm/metadata_logs_otc/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc/static/basic.output.yaml @@ -19,11 +19,6 @@ data: add_timestamp: true timestamp_key: timestamp log_format: json - metadata_attributes: - - _collector - - k8s.* - - pod_labels_.* - - namespace_labels_.* sending_queue: enabled: true persistent_storage_enabled: true @@ -36,8 +31,6 @@ data: add_timestamp: true timestamp_key: timestamp log_format: json - metadata_attributes: - - _collector sending_queue: enabled: true persistent_storage_enabled: true @@ -129,13 +122,11 @@ data: - k8s.container.name - k8s.namespace.name - k8s.pod.name + - _collector groupbyattrs/systemd: keys: - - _SYSTEMD_UNIT - - _HOSTNAME - - SYSLOG_FACILITY - - PRIORITY - _sourceName + - _collector k8s_tagger: extract: annotations: diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml index fb9082caa8..9da20d5eae 100644 --- a/tests/helm/metadata_logs_otc/static/templates.output.yaml +++ b/tests/helm/metadata_logs_otc/static/templates.output.yaml @@ -19,11 +19,6 @@ data: add_timestamp: true timestamp_key: timestamp log_format: json - metadata_attributes: - - _collector - - k8s.* - - pod_labels_.* - - namespace_labels_.* sending_queue: enabled: true persistent_storage_enabled: true @@ -36,8 +31,6 @@ data: add_timestamp: true timestamp_key: timestamp log_format: json - metadata_attributes: - - _collector sending_queue: enabled: true persistent_storage_enabled: true @@ -129,13 +122,11 @@ data: - k8s.container.name - k8s.namespace.name - k8s.pod.name + - _collector groupbyattrs/systemd: keys: - - _SYSTEMD_UNIT - - _HOSTNAME - - SYSLOG_FACILITY - - PRIORITY - _sourceName + - _collector k8s_tagger: extract: annotations: diff --git a/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml b/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml index 87b857b653..51b9e5cbf6 100644 --- a/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml @@ -65,7 +65,7 @@ spec: priorityClassName: "prio" containers: - name: otelcol - image: public.ecr.aws/sumologic/sumologic-otel-collector:0.47.0-sumo-0 + image: public.ecr.aws/sumologic/sumologic-otel-collector:0.49.0-sumo-0 imagePullPolicy: IfNotPresent args: - --config=/etc/otel/config.yaml diff --git a/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml b/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml index 4529a63d2b..b42c0e8e56 100644 --- a/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml +++ b/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml @@ -65,7 +65,7 @@ spec: priorityClassName: "prio" containers: - name: otelcol - image: public.ecr.aws/sumologic/sumologic-otel-collector:0.47.0-sumo-0 + image: public.ecr.aws/sumologic/sumologic-otel-collector:0.49.0-sumo-0 imagePullPolicy: IfNotPresent args: - --config=/etc/otel/config.yaml diff --git a/tests/helm/tracing/static/collection-monitoring-false.output.yaml b/tests/helm/tracing/static/collection-monitoring-false.output.yaml index 36c36235fe..1c7c95c01f 100644 --- a/tests/helm/tracing/static/collection-monitoring-false.output.yaml +++ b/tests/helm/tracing/static/collection-monitoring-false.output.yaml @@ -20,9 +20,6 @@ data: endpoint: ${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE} log_format: text max_request_body_size: 1048576 - metadata_attributes: - - k8s.* - - host.name metric_format: prometheus retry_on_failure: enabled: true diff --git a/tests/helm/tracing/static/simple.output.yaml b/tests/helm/tracing/static/simple.output.yaml index b3821b6c88..47ca46dc3d 100644 --- a/tests/helm/tracing/static/simple.output.yaml +++ b/tests/helm/tracing/static/simple.output.yaml @@ -20,9 +20,6 @@ data: endpoint: ${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE} log_format: text max_request_body_size: 1048576 - metadata_attributes: - - k8s.* - - host.name metric_format: prometheus retry_on_failure: enabled: true diff --git a/vagrant/values.yaml b/vagrant/values.yaml index d4b62c6da5..391403b045 100644 --- a/vagrant/values.yaml +++ b/vagrant/values.yaml @@ -186,9 +186,6 @@ metadata: - action: extract key: fluent.tag pattern: ^containers\.var\.log\.pods\.(?P[^_]+)_(?P[^_]+)_(?P[a-f0-9\-]{36})\.(?P[^\._]+)\.(?P\d+)\.log$ - - action: insert - key: k8s.pod.uid - from_attribute: k8s_uid - action: delete key: k8s_uid - action: delete From 558c598831b02f844789b82634dc5291a7305ad7 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Thu, 28 Apr 2022 08:55:36 +0200 Subject: [PATCH 2/9] chore(changelog): update Signed-off-by: Dominik Rosiek --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c38010d0e..5e5f76f4c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - chore: change Fluent Bit image to `public.ecr.aws/sumologic/fluent-bit:1.6.10-sumo-2`, it is Fluent Bit 1.6.10 with updated dependencies, image repository: https://github.com/SumoLogic/fluent-bit-docker-image [#2254][#2254] +- feat(metadata/otc): upgrade to v0.49.0-sumo-0 [#2251][#2251] ### Fixed @@ -36,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#2246]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2246 [#2254]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2254 [#2255]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2255 +[#2251]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2251 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.6.0...main ## [v2.7.0][v2.7.0] From 8f48e283998fa9b6d8db5fc97f391c23fb2ad1f7 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Thu, 28 Apr 2022 09:18:33 +0200 Subject: [PATCH 3/9] feat(metrics/opentelemetry): remove http_listener_v2_path in routing processor Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 65d3e5c623..33f91be1e0 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3689,7 +3689,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - routing_atttribute_to_drop: http_listener_v2_path sumologic/apiserver: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_APISERVER_METRICS_SOURCE} @@ -3701,7 +3700,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - routing_atttribute_to_drop: http_listener_v2_path sumologic/control_plane: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_CONTROL_PLANE_METRICS_SOURCE} @@ -3713,7 +3711,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - routing_atttribute_to_drop: http_listener_v2_path sumologic/controller: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_CONTROLLER_METRICS_SOURCE} @@ -3725,7 +3722,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - routing_atttribute_to_drop: http_listener_v2_path sumologic/kubelet: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_KUBELET_METRICS_SOURCE} @@ -3737,7 +3733,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - routing_atttribute_to_drop: http_listener_v2_path sumologic/node: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_NODE_METRICS_SOURCE} @@ -3749,7 +3744,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - routing_atttribute_to_drop: http_listener_v2_path sumologic/scheduler: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_SCHEDULER_METRICS_SOURCE} @@ -3761,7 +3755,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - routing_atttribute_to_drop: http_listener_v2_path sumologic/state: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_STATE_METRICS_SOURCE} @@ -3773,7 +3766,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - routing_atttribute_to_drop: http_listener_v2_path processors: ## Configuration for Metrics Transform Processor ## ref: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.37.x/processor/metricstransformprocessor @@ -3834,6 +3826,7 @@ metadata: ## NOTE: add a feature to routingprocessor to drop the routing ## attribute to prevent it being sent to the exporters. from_attribute: http_listener_v2_path + drop_resource_routing_attribute: true attribute_source: resource default_exporters: - sumologic/default From e9d98fdc46cc57267f019ab228967fbd89f62ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Thu, 28 Apr 2022 11:50:57 +0200 Subject: [PATCH 4/9] fix(otelcol/systemd): fix systemd pipeline --- deploy/helm/sumologic/values.yaml | 78 +++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 13 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 33f91be1e0..dce7322574 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -4168,9 +4168,6 @@ metadata: - replicaSetName - serviceName - statefulSetName - annotations: - - tag_name: "pod_annotations_%s" - key: "*" namespace_labels: - tag_name: "namespace_labels_%s" key: "*" @@ -4227,6 +4224,63 @@ metadata: record_attributes: - key: _SYSTEMD_UNIT value: kubelet.service + filter/exclude_systemd_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: '{{ .Values.fluentd.logs.systemd.excludeFacilityRegex | quote }}' + filter/exclude_systemd_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: '{{ .Values.fluentd.logs.systemd.excludeHostRegex | quote }}' + filter/exclude_systemd_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: '{{ .Values.fluentd.logs.systemd.excludePriorityRegex | quote }}' + filter/exclude_systemd_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: '{{ .Values.fluentd.logs.systemd.excludeUnitRegex | quote }}' + filter/exclude_kubelet_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: '{{ .Values.fluentd.logs.kubelet.excludeFacilityRegex | quote }}' + filter/exclude_kubelet_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: '{{ .Values.fluentd.logs.kubelet.excludeHostRegex | quote }}' + filter/exclude_kubelet_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: '{{ .Values.fluentd.logs.kubelet.excludePriorityRegex | quote }}' + filter/exclude_kubelet_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: '{{ .Values.fluentd.logs.kubelet.excludeUnitRegex | quote }}' + groupbyattrs/systemd: keys: - _sourceName @@ -4238,11 +4292,6 @@ metadata: source_category: '{{ .Values.fluentd.logs.systemd.sourceCategory | quote }}' source_category_prefix: '{{ .Values.fluentd.logs.systemd.sourceCategoryPrefix | quote }}' source_category_replace_dash: '{{ .Values.fluentd.logs.systemd.sourceCategoryReplaceDash | quote }}' - exclude: - SYSLOG_FACILITY: '{{ .Values.fluentd.logs.systemd.excludeFacilityRegex | quote }}' - _HOSTNAME: '{{ .Values.fluentd.logs.systemd.excludeHostRegex | quote }}' - PRIORITY: '{{ .Values.fluentd.logs.systemd.excludePriorityRegex | quote }}' - _SYSTEMD_UNIT: '{{ .Values.fluentd.logs.systemd.excludeUnitRegex | quote }}' ## kubelet related processors filter/include_kubelet: @@ -4259,11 +4308,6 @@ metadata: source_category: '{{ .Values.fluentd.logs.kubelet.sourceCategory | quote }}' source_category_prefix: '{{ .Values.fluentd.logs.kubelet.sourceCategoryPrefix | quote }}' source_category_replace_dash: '{{ .Values.fluentd.logs.kubelet.sourceCategoryReplaceDash | quote }}' - exclude: - SYSLOG_FACILITY: '{{ .Values.fluentd.logs.kubelet.excludeFacilityRegex | quote }}' - _HOSTNAME: '{{ .Values.fluentd.logs.kubelet.excludeHostRegex | quote }}' - PRIORITY: '{{ .Values.fluentd.logs.kubelet.excludePriorityRegex | quote }}' - _SYSTEMD_UNIT: '{{ .Values.fluentd.logs.kubelet.excludeUnitRegex | quote }}' service: telemetry: @@ -4315,6 +4359,10 @@ metadata: - filter/include_fluent_tag_host - filter/include_systemd - filter/exclude_kubelet + - filter/exclude_systemd_syslog + - filter/exclude_systemd_hostname + - filter/exclude_systemd_priority + - filter/exclude_systemd_unit - groupbyattrs/systemd - source/systemd - attributes/remove_fluent_tag @@ -4328,6 +4376,10 @@ metadata: - memory_limiter - filter/include_fluent_tag_host - filter/include_kubelet + - filter/exclude_kubelet_syslog + - filter/exclude_kubelet_hostname + - filter/exclude_kubelet_priority + - filter/exclude_kubelet_unit - groupbyattrs/systemd - source/kubelet - attributes/remove_fluent_tag From 227fd0b573ad3a4eae09e9a4b18dd8ef3d9a0ffb Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Fri, 29 Apr 2022 11:45:54 +0200 Subject: [PATCH 5/9] tests(templates): update tests for metadata_logs_otc Signed-off-by: Dominik Rosiek --- .../static/basic.output.yaml | 77 +++++++++++++++---- .../static/templates.output.yaml | 77 +++++++++++++++---- 2 files changed, 128 insertions(+), 26 deletions(-) diff --git a/tests/helm/metadata_logs_otc/static/basic.output.yaml b/tests/helm/metadata_logs_otc/static/basic.output.yaml index 47af0bb0b0..2f12294886 100644 --- a/tests/helm/metadata_logs_otc/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc/static/basic.output.yaml @@ -88,6 +88,62 @@ data: record_attributes: - key: _SYSTEMD_UNIT value: kubelet.service + filter/exclude_kubelet_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: "" + filter/exclude_kubelet_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: "" + filter/exclude_kubelet_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: "" + filter/exclude_kubelet_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: "" + filter/exclude_systemd_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: "" + filter/exclude_systemd_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: "" + filter/exclude_systemd_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: "" + filter/exclude_systemd_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: "" filter/include_fluent_tag_containers: logs: include: @@ -129,9 +185,6 @@ data: - _collector k8s_tagger: extract: - annotations: - - key: '*' - tag_name: pod_annotations_%s delimiter: _ labels: - key: '*' @@ -187,11 +240,6 @@ data: source_name: '%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}' source/kubelet: collector: "kubernetes" - exclude: - _HOSTNAME: "" - _SYSTEMD_UNIT: "" - PRIORITY: "" - SYSLOG_FACILITY: "" source_category: "kubelet" source_category_prefix: "kubernetes/" source_category_replace_dash: "/" @@ -199,11 +247,6 @@ data: source_name: "k8s_kubelet" source/systemd: collector: "kubernetes" - exclude: - _HOSTNAME: "" - _SYSTEMD_UNIT: "" - PRIORITY: "" - SYSLOG_FACILITY: "" source_category: "system" source_category_prefix: "kubernetes/" source_category_replace_dash: "/" @@ -244,6 +287,10 @@ data: - memory_limiter - filter/include_fluent_tag_host - filter/include_kubelet + - filter/exclude_kubelet_syslog + - filter/exclude_kubelet_hostname + - filter/exclude_kubelet_priority + - filter/exclude_kubelet_unit - groupbyattrs/systemd - source/kubelet - attributes/remove_fluent_tag @@ -258,6 +305,10 @@ data: - filter/include_fluent_tag_host - filter/include_systemd - filter/exclude_kubelet + - filter/exclude_systemd_syslog + - filter/exclude_systemd_hostname + - filter/exclude_systemd_priority + - filter/exclude_systemd_unit - groupbyattrs/systemd - source/systemd - attributes/remove_fluent_tag diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml index 9da20d5eae..89ee15c976 100644 --- a/tests/helm/metadata_logs_otc/static/templates.output.yaml +++ b/tests/helm/metadata_logs_otc/static/templates.output.yaml @@ -88,6 +88,62 @@ data: record_attributes: - key: _SYSTEMD_UNIT value: kubelet.service + filter/exclude_kubelet_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: "my_kubelet_excludeHostRegex" + filter/exclude_kubelet_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: "my_kubelet_excludePriorityRegex" + filter/exclude_kubelet_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: "my_kubelet_excludeFacilityRegex" + filter/exclude_kubelet_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: "my_kubelet_excludeUnitRegex" + filter/exclude_systemd_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: "my_systemd_excludeHostRegex" + filter/exclude_systemd_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: "my_systemd_excludePriorityRegex" + filter/exclude_systemd_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: "my_systemd_excludeFacilityRegex" + filter/exclude_systemd_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: "my_systemd_excludeUnitRegex" filter/include_fluent_tag_containers: logs: include: @@ -129,9 +185,6 @@ data: - _collector k8s_tagger: extract: - annotations: - - key: '*' - tag_name: pod_annotations_%s delimiter: _ labels: - key: '*' @@ -187,11 +240,6 @@ data: source_name: '%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}' source/kubelet: collector: "my_collectorName" - exclude: - _HOSTNAME: "my_kubelet_excludeHostRegex" - _SYSTEMD_UNIT: "my_kubelet_excludeUnitRegex" - PRIORITY: "my_kubelet_excludePriorityRegex" - SYSLOG_FACILITY: "my_kubelet_excludeFacilityRegex" source_category: "kubelet" source_category_prefix: "my_kubelet_sourceCategoryPrefix" source_category_replace_dash: "my_kubelet_sourceCategoryReplaceDash" @@ -199,11 +247,6 @@ data: source_name: "k8s_kubelet" source/systemd: collector: "my_collectorName" - exclude: - _HOSTNAME: "my_systemd_excludeHostRegex" - _SYSTEMD_UNIT: "my_systemd_excludeUnitRegex" - PRIORITY: "my_systemd_excludePriorityRegex" - SYSLOG_FACILITY: "my_systemd_excludeFacilityRegex" source_category: "system" source_category_prefix: "my_systemd_sourceCategoryPrefix" source_category_replace_dash: "my_systemd_sourceCategoryReplaceDash" @@ -244,6 +287,10 @@ data: - memory_limiter - filter/include_fluent_tag_host - filter/include_kubelet + - filter/exclude_kubelet_syslog + - filter/exclude_kubelet_hostname + - filter/exclude_kubelet_priority + - filter/exclude_kubelet_unit - groupbyattrs/systemd - source/kubelet - attributes/remove_fluent_tag @@ -258,6 +305,10 @@ data: - filter/include_fluent_tag_host - filter/include_systemd - filter/exclude_kubelet + - filter/exclude_systemd_syslog + - filter/exclude_systemd_hostname + - filter/exclude_systemd_priority + - filter/exclude_systemd_unit - groupbyattrs/systemd - source/systemd - attributes/remove_fluent_tag From 0135eb1f4eb0687106d7d9ee9c439da1834d8cf1 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Fri, 29 Apr 2022 15:47:13 +0200 Subject: [PATCH 6/9] fix: fix syslog and kubelet filtering according to integration tests Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 19 +++++++++++-------- .../static/basic.output.yaml | 19 +++++++++++-------- .../static/templates.output.yaml | 3 +++ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index dce7322574..aeb32b894d 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -4168,6 +4168,9 @@ metadata: - replicaSetName - serviceName - statefulSetName + annotations: + - tag_name: "pod_annotations_%s" + key: "*" namespace_labels: - tag_name: "namespace_labels_%s" key: "*" @@ -4230,56 +4233,56 @@ metadata: match_type: regexp record_attributes: - key: SYSLOG_FACILITY - value: '{{ .Values.fluentd.logs.systemd.excludeFacilityRegex | quote }}' + value: '{{ .Values.fluentd.logs.systemd.excludeFacilityRegex | default "$^" | quote }}' filter/exclude_systemd_hostname: logs: exclude: match_type: regexp record_attributes: - key: _HOSTNAME - value: '{{ .Values.fluentd.logs.systemd.excludeHostRegex | quote }}' + value: '{{ .Values.fluentd.logs.systemd.excludeHostRegex | default "$^" | quote }}' filter/exclude_systemd_priority: logs: exclude: match_type: regexp record_attributes: - key: PRIORITY - value: '{{ .Values.fluentd.logs.systemd.excludePriorityRegex | quote }}' + value: '{{ .Values.fluentd.logs.systemd.excludePriorityRegex | default "$^" | quote }}' filter/exclude_systemd_unit: logs: exclude: match_type: regexp record_attributes: - key: _SYSTEMD_UNIT - value: '{{ .Values.fluentd.logs.systemd.excludeUnitRegex | quote }}' + value: '{{ .Values.fluentd.logs.systemd.excludeUnitRegex | default "$^" | quote }}' filter/exclude_kubelet_syslog: logs: exclude: match_type: regexp record_attributes: - key: SYSLOG_FACILITY - value: '{{ .Values.fluentd.logs.kubelet.excludeFacilityRegex | quote }}' + value: '{{ .Values.fluentd.logs.kubelet.excludeFacilityRegex | default "$^" | quote }}' filter/exclude_kubelet_hostname: logs: exclude: match_type: regexp record_attributes: - key: _HOSTNAME - value: '{{ .Values.fluentd.logs.kubelet.excludeHostRegex | quote }}' + value: '{{ .Values.fluentd.logs.kubelet.excludeHostRegex | default "$^" | quote }}' filter/exclude_kubelet_priority: logs: exclude: match_type: regexp record_attributes: - key: PRIORITY - value: '{{ .Values.fluentd.logs.kubelet.excludePriorityRegex | quote }}' + value: '{{ .Values.fluentd.logs.kubelet.excludePriorityRegex | default "$^" | quote }}' filter/exclude_kubelet_unit: logs: exclude: match_type: regexp record_attributes: - key: _SYSTEMD_UNIT - value: '{{ .Values.fluentd.logs.kubelet.excludeUnitRegex | quote }}' + value: '{{ .Values.fluentd.logs.kubelet.excludeUnitRegex | default "$^" | quote }}' groupbyattrs/systemd: keys: diff --git a/tests/helm/metadata_logs_otc/static/basic.output.yaml b/tests/helm/metadata_logs_otc/static/basic.output.yaml index 2f12294886..e8f1fb61b2 100644 --- a/tests/helm/metadata_logs_otc/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc/static/basic.output.yaml @@ -94,56 +94,56 @@ data: match_type: regexp record_attributes: - key: _HOSTNAME - value: "" + value: "$^" filter/exclude_kubelet_priority: logs: exclude: match_type: regexp record_attributes: - key: PRIORITY - value: "" + value: "$^" filter/exclude_kubelet_syslog: logs: exclude: match_type: regexp record_attributes: - key: SYSLOG_FACILITY - value: "" + value: "$^" filter/exclude_kubelet_unit: logs: exclude: match_type: regexp record_attributes: - key: _SYSTEMD_UNIT - value: "" + value: "$^" filter/exclude_systemd_hostname: logs: exclude: match_type: regexp record_attributes: - key: _HOSTNAME - value: "" + value: "$^" filter/exclude_systemd_priority: logs: exclude: match_type: regexp record_attributes: - key: PRIORITY - value: "" + value: "$^" filter/exclude_systemd_syslog: logs: exclude: match_type: regexp record_attributes: - key: SYSLOG_FACILITY - value: "" + value: "$^" filter/exclude_systemd_unit: logs: exclude: match_type: regexp record_attributes: - key: _SYSTEMD_UNIT - value: "" + value: "$^" filter/include_fluent_tag_containers: logs: include: @@ -185,6 +185,9 @@ data: - _collector k8s_tagger: extract: + annotations: + - key: '*' + tag_name: pod_annotations_%s delimiter: _ labels: - key: '*' diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml index 89ee15c976..bb7f63a70b 100644 --- a/tests/helm/metadata_logs_otc/static/templates.output.yaml +++ b/tests/helm/metadata_logs_otc/static/templates.output.yaml @@ -185,6 +185,9 @@ data: - _collector k8s_tagger: extract: + annotations: + - key: '*' + tag_name: pod_annotations_%s delimiter: _ labels: - key: '*' From 8b1f36834f2d002be8cc7581c22928e9d5b647c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Wed, 4 May 2022 13:47:09 +0200 Subject: [PATCH 7/9] fix: drop Pod annotation attributes for container logs --- deploy/helm/sumologic/values.yaml | 6 ++++++ tests/helm/metadata_logs_otc/static/basic.output.yaml | 5 +++++ tests/helm/metadata_logs_otc/static/templates.output.yaml | 5 +++++ 3 files changed, 16 insertions(+) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index aeb32b894d..6679d03613 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -4141,6 +4141,10 @@ metadata: - action: upsert key: k8s.pod.hostname from_attribute: k8s.node.name + resource/drop_annotations: + attributes: + - pattern: ^pod_annotations_.* + action: delete groupbyattrs/containers: keys: - k8s.container.id @@ -4331,6 +4335,7 @@ metadata: - groupbyattrs/containers - k8s_tagger - source/containers + - resource/drop_annotations - attributes/remove_fluent_tag - resource/containers_copy_node_to_host - batch @@ -4350,6 +4355,7 @@ metadata: # - groupbyattrs/containers # - k8s_tagger # - source/containers + # - resource/drop_annotations # - resource/containers_copy_node_to_host # - batch # exporters: diff --git a/tests/helm/metadata_logs_otc/static/basic.output.yaml b/tests/helm/metadata_logs_otc/static/basic.output.yaml index e8f1fb61b2..840a8b9f92 100644 --- a/tests/helm/metadata_logs_otc/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc/static/basic.output.yaml @@ -222,6 +222,10 @@ data: - action: upsert from_attribute: k8s.node.name key: k8s.pod.hostname + resource/drop_annotations: + attributes: + - action: delete + pattern: ^pod_annotations_.* source/containers: annotation_prefix: pod_annotations_ collector: "kubernetes" @@ -278,6 +282,7 @@ data: - groupbyattrs/containers - k8s_tagger - source/containers + - resource/drop_annotations - attributes/remove_fluent_tag - resource/containers_copy_node_to_host - batch diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml index bb7f63a70b..cc7efd30e6 100644 --- a/tests/helm/metadata_logs_otc/static/templates.output.yaml +++ b/tests/helm/metadata_logs_otc/static/templates.output.yaml @@ -222,6 +222,10 @@ data: - action: upsert from_attribute: k8s.node.name key: k8s.pod.hostname + resource/drop_annotations: + attributes: + - action: delete + pattern: ^pod_annotations_.* source/containers: annotation_prefix: pod_annotations_ collector: "my_collectorName" @@ -278,6 +282,7 @@ data: - groupbyattrs/containers - k8s_tagger - source/containers + - resource/drop_annotations - attributes/remove_fluent_tag - resource/containers_copy_node_to_host - batch From afd246a2ff6c8d355c69357837c5d4a62ad10ad0 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Wed, 4 May 2022 15:42:59 +0200 Subject: [PATCH 8/9] chore(otelcol): upgrade to v0.50.0-sumo-0 Signed-off-by: Dominik Rosiek --- CHANGELOG.md | 2 +- deploy/docs/Best_Practices.md | 2 +- deploy/docs/fluentd_otc_comparison.md | 38 +++++++++---------- deploy/helm/sumologic/values.yaml | 4 +- .../static/basic.output.yaml | 2 +- .../static/basic.output.yaml | 2 +- .../static/basic.output.yaml | 2 +- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5f76f4c7..f7c751b1dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - chore: change Fluent Bit image to `public.ecr.aws/sumologic/fluent-bit:1.6.10-sumo-2`, it is Fluent Bit 1.6.10 with updated dependencies, image repository: https://github.com/SumoLogic/fluent-bit-docker-image [#2254][#2254] -- feat(metadata/otc): upgrade to v0.49.0-sumo-0 [#2251][#2251] +- feat(metadata/otc): upgrade to v0.50.0-sumo-0 [#2251][#2251] ### Fixed diff --git a/deploy/docs/Best_Practices.md b/deploy/docs/Best_Practices.md index e590b7036e..7a778621c6 100644 --- a/deploy/docs/Best_Practices.md +++ b/deploy/docs/Best_Practices.md @@ -1055,7 +1055,7 @@ For [sumologic exporter][sumologic_exporter]: **because once filled in PVC never reduces its fill.** [batch_processor]: https://github.com/open-telemetry/opentelemetry-collector/tree/v0.47.0/processor/batchprocessor#batch-processor -[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/exporter/sumologicexporter#sumo-logic-exporter +[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/exporter/sumologicexporter#sumo-logic-exporter [filling_up_alert]: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup/ ### Compaction diff --git a/deploy/docs/fluentd_otc_comparison.md b/deploy/docs/fluentd_otc_comparison.md index f91aa3e3a5..3891390d1f 100644 --- a/deploy/docs/fluentd_otc_comparison.md +++ b/deploy/docs/fluentd_otc_comparison.md @@ -54,10 +54,10 @@ Additional behavior: [fields_base]: https://github.com/SumoLogic/fluentd-output-sumologic/blob/1.7.2/lib/fluent/plugin/out_sumologic.rb#L284-L285 [fluentd_output_plugin]: https://github.com/sumologic/fluentd-output-sumologic/tree/1.7.2#configuration -[otelcol_basic_confg]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/docs/Configuration.md#basic-configuration -[otelcol_proxy]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/docs/Configuration.md#proxy-support -[otelcol_source_config]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#config -[otelcol_sumologic_config]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter +[otelcol_basic_confg]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/docs/Configuration.md#basic-configuration +[otelcol_proxy]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/docs/Configuration.md#proxy-support +[otelcol_source_config]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#config +[otelcol_sumologic_config]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter [otelocl_tls_config]: https://github.com/open-telemetry/opentelemetry-collector/blob/v0.47.0/config/configtls/README.md#tls--mtls-configuration [resource_processor]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.47.0/processor/resourceprocessor#resource-processor [source_category_precedence]: https://github.com/SumoLogic/fluentd-output-sumologic/blob/1.7.2/lib/fluent/plugin/out_sumologic.rb#L278-L279 @@ -70,7 +70,7 @@ In order to receive prometheus data and for their initial processing [telegrafre It should cover [fluent-plugin-datapoint][fluent_plugin_datapoint] functionality and more. [fluent_plugin_datapoint]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-datapoint -[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/receiver/telegrafreceiver +[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/receiver/telegrafreceiver ### fluent-plugin-protobuf @@ -78,7 +78,7 @@ In order to receive prometheus data and for their initial processing [telegrafre It should cover [fluent_plugin_protobuf][fluent_plugin_protobuf] functionality and more. [fluent_plugin_protobuf]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-protobuf -[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/receiver/telegrafreceiver +[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/receiver/telegrafreceiver ### fluent-plugin-prometheus-format @@ -162,17 +162,17 @@ Sanitized pod name is name portion of the pod. Please consider following example [kube_daemonset]: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/ [kube_deployment]: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ [kube_statefulset]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset -[otelcol_annotations]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#pod-annotations +[otelcol_annotations]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#pod-annotations [otelcol_annotations_exclude]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/blob/v1.12.2-sumo-4/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb#L171-L183 -[otelcol_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/exporter/sumologicexporter#source-templates -[otelcol_undefined]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/processor/sourceprocessor/attribute_filler.go#L113-L123 -[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations -[source_filtering]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#filtering-section -[source_keys]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#keys-section -[source_processor]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#config -[source_processor_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#name-translation-and-template-keys +[otelcol_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/exporter/sumologicexporter#source-templates +[otelcol_undefined]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/processor/sourceprocessor/attribute_filler.go#L113-L123 +[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations +[source_filtering]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#filtering-section +[source_keys]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#keys-section +[source_processor]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#config +[source_processor_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#name-translation-and-template-keys [sumo_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-kubernetes-sumologic#fluent-plugin-kubernetes-sumologic -[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter +[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter ### fluent-plugin-kubernetes-metadata-filter @@ -190,8 +190,8 @@ Sanitized pod name is name portion of the pod. Please consider following example | `cache_ttl` | N/A | [fluent_plugin_k8s_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-kubernetes-metadata-filter#configuration -[k8sprocessor]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/processor/k8sprocessor -[k8sprocessor_field_extract]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/k8sprocessor#field-extract-config +[k8sprocessor]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/processor/k8sprocessor +[k8sprocessor_field_extract]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/k8sprocessor#field-extract-config [kubeconfig_auth_type]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.47.0/internal/k8sconfig/config.go#L53-L60 ### fluent-plugin-enhance-k8s-metadata @@ -209,7 +209,7 @@ Sanitized pod name is name portion of the pod. Please consider following example | `data_type` | N/A | [fluent_plugin_enhance_k8s_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-enhance-k8s-metadata#configuration -[pod_association]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.49.0-sumo-0/pkg/processor/k8sprocessor/doc.go#L17-L46 +[pod_association]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/processor/k8sprocessor/doc.go#L17-L46 ### fluent-plugin-events @@ -358,7 +358,7 @@ Events are not supported by `Opentelemetry Collector` [filter_processor]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.47.0/processor/filterprocessor#filter-processor [readme]: ../helm/sumologic/README.md -[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.49.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations +[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations [persistent_queue]: https://github.com/open-telemetry/opentelemetry-collector/tree/release/v0.37.x/exporter/exporterhelper#persistent-queue [file_storage_extension]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.37.x/extension/storage/filestorage diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 6679d03613..9d0860811d 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3581,7 +3581,7 @@ metadata: ## Configure image for Opentelemetry Collector (for logs and metrics) image: repository: public.ecr.aws/sumologic/sumologic-otel-collector - tag: 0.49.0-sumo-0 + tag: 0.50.0-sumo-0 pullPolicy: IfNotPresent securityContext: @@ -4492,7 +4492,7 @@ otelgateway: podAnnotations: {} image: repository: public.ecr.aws/sumologic/sumologic-otel-collector - tag: 0.49.0-sumo-0 + tag: 0.50.0-sumo-0 pullPolicy: IfNotPresent livenessProbe: periodSeconds: 15 diff --git a/tests/helm/logs_otc_daemonset/static/basic.output.yaml b/tests/helm/logs_otc_daemonset/static/basic.output.yaml index 9656ad04a1..1492c5f54b 100644 --- a/tests/helm/logs_otc_daemonset/static/basic.output.yaml +++ b/tests/helm/logs_otc_daemonset/static/basic.output.yaml @@ -30,7 +30,7 @@ spec: containers: - args: - --config=/etc/otelcol/config.yaml - image: public.ecr.aws/sumologic/sumologic-otel-collector:0.49.0-sumo-0 + image: public.ecr.aws/sumologic/sumologic-otel-collector:0.50.0-sumo-0 imagePullPolicy: IfNotPresent name: otelcol livenessProbe: diff --git a/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml b/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml index 51b9e5cbf6..db7e0bc168 100644 --- a/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml @@ -65,7 +65,7 @@ spec: priorityClassName: "prio" containers: - name: otelcol - image: public.ecr.aws/sumologic/sumologic-otel-collector:0.49.0-sumo-0 + image: public.ecr.aws/sumologic/sumologic-otel-collector:0.50.0-sumo-0 imagePullPolicy: IfNotPresent args: - --config=/etc/otel/config.yaml diff --git a/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml b/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml index b42c0e8e56..7dbb045225 100644 --- a/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml +++ b/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml @@ -65,7 +65,7 @@ spec: priorityClassName: "prio" containers: - name: otelcol - image: public.ecr.aws/sumologic/sumologic-otel-collector:0.49.0-sumo-0 + image: public.ecr.aws/sumologic/sumologic-otel-collector:0.50.0-sumo-0 imagePullPolicy: IfNotPresent args: - --config=/etc/otel/config.yaml From 947f6e145dc03f1c8d494182fa3ba48bad0bf22f Mon Sep 17 00:00:00 2001 From: Dominik Rosiek <58699848+sumo-drosiek@users.noreply.github.com> Date: Wed, 4 May 2022 15:51:14 +0200 Subject: [PATCH 9/9] fix: revert unwanted change --- tests/helm/logs_otc_daemonset/static/basic.output.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helm/logs_otc_daemonset/static/basic.output.yaml b/tests/helm/logs_otc_daemonset/static/basic.output.yaml index 1492c5f54b..9656ad04a1 100644 --- a/tests/helm/logs_otc_daemonset/static/basic.output.yaml +++ b/tests/helm/logs_otc_daemonset/static/basic.output.yaml @@ -30,7 +30,7 @@ spec: containers: - args: - --config=/etc/otelcol/config.yaml - image: public.ecr.aws/sumologic/sumologic-otel-collector:0.50.0-sumo-0 + image: public.ecr.aws/sumologic/sumologic-otel-collector:0.49.0-sumo-0 imagePullPolicy: IfNotPresent name: otelcol livenessProbe: