From 845f42885e75ecd59f979acce21c95cdaf9d64d4 Mon Sep 17 00:00:00 2001 From: Adam Boguszewski Date: Fri, 27 Jan 2023 17:34:15 +0100 Subject: [PATCH] feat(otelcol): add max size in batch processors --- CHANGELOG.md | 2 ++ .../helm/sumologic/conf/logs/collector/otelcol/config.yaml | 1 + deploy/helm/sumologic/conf/logs/otelcol/config.yaml | 2 ++ deploy/helm/sumologic/conf/metrics/otelcol/config.yaml | 2 ++ deploy/helm/sumologic/values.yaml | 6 ++++-- tests/helm/logs_otc/static/basic.output.yaml | 1 + tests/helm/logs_otc/static/options.output.yaml | 1 + tests/helm/metadata_logs_otc/static/otel.output.yaml | 1 + tests/helm/metadata_logs_otc/static/templates.output.yaml | 1 + .../static/additional_endpoints.output.yaml | 1 + tests/helm/metadata_metrics_otc/static/basic.output.yaml | 1 + .../static/traces-gateway-disabled.output.yaml | 1 + .../static/traces-gateway-enabled.output.yaml | 1 + .../static/traces-gateway-true.output.yaml | 1 + 14 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b2022587..3538601967 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - chore: add support for EKS 1.24 [#2831] - chore: add support for GKE 1.24 [#2832] +- feat(otelcol): add max size in batch processors [#2839] [#2831]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2831 [#2832]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2832 +[#2839]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2839 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v3.0.0...main ## [v3.0.0] diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml index 6c48d4a93d..df6c446454 100644 --- a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml @@ -269,6 +269,7 @@ exporters: processors: batch: send_batch_size: 1000 + send_batch_max_size: 2000 timeout: 1s ## copy _SYSTEMD_UNIT, SYSLOG_FACILITY, _HOSTNAME and PRIORITY from body to attributes ## so they can be used by metadata processors same way like for fluentd diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml index 6171082c98..0352bb1ed6 100644 --- a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml @@ -88,6 +88,8 @@ processors: batch: ## Number of spans after which a batch will be sent regardless of time send_batch_size: 1_024 + ## Maximum number of spans sent at once + send_batch_max_size: 2_048 ## Time duration after which a batch will be sent regardless of size timeout: 1s resource/add_cluster: diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml index 7cf1c815b9..cf05e501f2 100644 --- a/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml @@ -271,6 +271,8 @@ processors: batch: ## Number of spans after which a batch will be sent regardless of time send_batch_size: 1_024 + ## Maximum number of spans sent at once + send_batch_max_size: 2_048 ## Time duration after which a batch will be sent regardless of size timeout: 1s ## Configuration for Kubernetes Processor diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 10bd8039a3..2b644b51f0 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3604,10 +3604,10 @@ otelcolInstrumentation: batch: ## Number of spans after which a batch will be sent regardless of time send_batch_size: 256 + ## Never more than this many spans are being sent in a batch + send_batch_max_size: 512 ## Time duration after which a batch will be sent regardless of size timeout: 5s - ## Never more than this many spans are being sent in a batch - # send_batch_max_size: 512 extensions: health_check: {} memory_ballast: @@ -4095,6 +4095,8 @@ tracesGateway: batch: ## Number of spans after which a batch will be sent regardless of time send_batch_size: 256 + ## Maximum number of spans sent at once + send_batch_max_size: 512 ## Time duration after which a batch will be sent regardless of size timeout: 5s ## Never more than this many spans are being sent in a batch diff --git a/tests/helm/logs_otc/static/basic.output.yaml b/tests/helm/logs_otc/static/basic.output.yaml index 8aac9035d1..dd745c1be4 100644 --- a/tests/helm/logs_otc/static/basic.output.yaml +++ b/tests/helm/logs_otc/static/basic.output.yaml @@ -26,6 +26,7 @@ data: pprof: {} processors: batch: + send_batch_max_size: 2000 send_batch_size: 1000 timeout: 1s logstransform/systemd: diff --git a/tests/helm/logs_otc/static/options.output.yaml b/tests/helm/logs_otc/static/options.output.yaml index d6ed8eab1b..8a23a87606 100644 --- a/tests/helm/logs_otc/static/options.output.yaml +++ b/tests/helm/logs_otc/static/options.output.yaml @@ -26,6 +26,7 @@ data: pprof: {} processors: batch: + send_batch_max_size: 2000 send_batch_size: 1000 timeout: 1s logstransform/systemd: diff --git a/tests/helm/metadata_logs_otc/static/otel.output.yaml b/tests/helm/metadata_logs_otc/static/otel.output.yaml index b1fa8e99fb..300e0ded0b 100644 --- a/tests/helm/metadata_logs_otc/static/otel.output.yaml +++ b/tests/helm/metadata_logs_otc/static/otel.output.yaml @@ -85,6 +85,7 @@ data: - action: delete key: fluent.tag batch: + send_batch_max_size: 2048 send_batch_size: 1024 timeout: 1s filter/exclude_kubelet: diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml index 849a240d8e..af7abfca21 100644 --- a/tests/helm/metadata_logs_otc/static/templates.output.yaml +++ b/tests/helm/metadata_logs_otc/static/templates.output.yaml @@ -85,6 +85,7 @@ data: - action: delete key: fluent.tag batch: + send_batch_max_size: 2048 send_batch_size: 1024 timeout: 1s filter/exclude_kubelet: diff --git a/tests/helm/metadata_metrics_otc/static/additional_endpoints.output.yaml b/tests/helm/metadata_metrics_otc/static/additional_endpoints.output.yaml index 72437d415c..80b4332cba 100644 --- a/tests/helm/metadata_metrics_otc/static/additional_endpoints.output.yaml +++ b/tests/helm/metadata_metrics_otc/static/additional_endpoints.output.yaml @@ -104,6 +104,7 @@ data: pprof: {} processors: batch: + send_batch_max_size: 2048 send_batch_size: 1024 timeout: 1s k8s_tagger: diff --git a/tests/helm/metadata_metrics_otc/static/basic.output.yaml b/tests/helm/metadata_metrics_otc/static/basic.output.yaml index 0adc38fbef..a7e886588d 100644 --- a/tests/helm/metadata_metrics_otc/static/basic.output.yaml +++ b/tests/helm/metadata_metrics_otc/static/basic.output.yaml @@ -104,6 +104,7 @@ data: pprof: {} processors: batch: + send_batch_max_size: 2048 send_batch_size: 1024 timeout: 1s k8s_tagger: diff --git a/tests/helm/otelcol-instrumentation-config/static/traces-gateway-disabled.output.yaml b/tests/helm/otelcol-instrumentation-config/static/traces-gateway-disabled.output.yaml index 0d0a48ad72..4c936f05a6 100644 --- a/tests/helm/otelcol-instrumentation-config/static/traces-gateway-disabled.output.yaml +++ b/tests/helm/otelcol-instrumentation-config/static/traces-gateway-disabled.output.yaml @@ -37,6 +37,7 @@ data: pprof: {} processors: batch: + send_batch_max_size: 512 send_batch_size: 256 timeout: 5s k8s_tagger: diff --git a/tests/helm/otelcol-instrumentation-config/static/traces-gateway-enabled.output.yaml b/tests/helm/otelcol-instrumentation-config/static/traces-gateway-enabled.output.yaml index 688831dfe7..c8211e49bc 100644 --- a/tests/helm/otelcol-instrumentation-config/static/traces-gateway-enabled.output.yaml +++ b/tests/helm/otelcol-instrumentation-config/static/traces-gateway-enabled.output.yaml @@ -37,6 +37,7 @@ data: pprof: {} processors: batch: + send_batch_max_size: 512 send_batch_size: 256 timeout: 5s k8s_tagger: diff --git a/tests/helm/traces-gateway-loadbalancing/static/traces-gateway-true.output.yaml b/tests/helm/traces-gateway-loadbalancing/static/traces-gateway-true.output.yaml index 3e30d152e0..fc86036c0c 100644 --- a/tests/helm/traces-gateway-loadbalancing/static/traces-gateway-true.output.yaml +++ b/tests/helm/traces-gateway-loadbalancing/static/traces-gateway-true.output.yaml @@ -29,6 +29,7 @@ data: pprof: {} processors: batch: + send_batch_max_size: 512 send_batch_size: 256 timeout: 5s memory_limiter: