feat(otelcol): add max size in batch processors

SumoLogic · Jan 30, 2023 · 845f428 · 845f428
1 parent 606ebfd
commit 845f428
Show file tree

Hide file tree

Showing 14 changed files with 20 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,9 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - chore: add support for EKS 1.24 [#2831]
 - chore: add support for GKE 1.24 [#2832]
+- feat(otelcol): add max size in batch processors [#2839]
 
 [#2831]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2831
 [#2832]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2832
+[#2839]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2839
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v3.0.0...main
 
 ## [v3.0.0]

diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml
@@ -269,6 +269,7 @@ exporters:
 processors:
   batch:
     send_batch_size: 1000
+    send_batch_max_size: 2000
     timeout: 1s
   ## copy _SYSTEMD_UNIT, SYSLOG_FACILITY, _HOSTNAME and PRIORITY from body to attributes
   ## so they can be used by metadata processors same way like for fluentd

diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml
@@ -88,6 +88,8 @@ processors:
   batch:
     ## Number of spans after which a batch will be sent regardless of time
     send_batch_size: 1_024
+    ## Maximum number of spans sent at once
+    send_batch_max_size: 2_048
     ## Time duration after which a batch will be sent regardless of size
     timeout: 1s
   resource/add_cluster:

diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml
@@ -271,6 +271,8 @@ processors:
   batch:
     ## Number of spans after which a batch will be sent regardless of time
     send_batch_size: 1_024
+    ## Maximum number of spans sent at once
+    send_batch_max_size: 2_048
     ## Time duration after which a batch will be sent regardless of size
     timeout: 1s
   ## Configuration for Kubernetes Processor

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -3604,10 +3604,10 @@ otelcolInstrumentation:
       batch:
         ## Number of spans after which a batch will be sent regardless of time
         send_batch_size: 256
+        ## Never more than this many spans are being sent in a batch
+        send_batch_max_size: 512
         ## Time duration after which a batch will be sent regardless of size
         timeout: 5s
-        ## Never more than this many spans are being sent in a batch
-        # send_batch_max_size: 512
     extensions:
       health_check: {}
       memory_ballast:
@@ -4095,6 +4095,8 @@ tracesGateway:
       batch:
         ## Number of spans after which a batch will be sent regardless of time
         send_batch_size: 256
+        ## Maximum number of spans sent at once
+        send_batch_max_size: 512
         ## Time duration after which a batch will be sent regardless of size
         timeout: 5s
         ## Never more than this many spans are being sent in a batch

diff --git a/tests/helm/logs_otc/static/basic.output.yaml b/tests/helm/logs_otc/static/basic.output.yaml
@@ -26,6 +26,7 @@ data:
       pprof: {}
     processors:
       batch:
+        send_batch_max_size: 2000
         send_batch_size: 1000
         timeout: 1s
       logstransform/systemd:

diff --git a/tests/helm/logs_otc/static/options.output.yaml b/tests/helm/logs_otc/static/options.output.yaml
@@ -26,6 +26,7 @@ data:
       pprof: {}
     processors:
       batch:
+        send_batch_max_size: 2000
         send_batch_size: 1000
         timeout: 1s
       logstransform/systemd:

diff --git a/tests/helm/metadata_logs_otc/static/otel.output.yaml b/tests/helm/metadata_logs_otc/static/otel.output.yaml
@@ -85,6 +85,7 @@ data:
         - action: delete
           key: fluent.tag
       batch:
+        send_batch_max_size: 2048
         send_batch_size: 1024
         timeout: 1s
       filter/exclude_kubelet:

diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml
@@ -85,6 +85,7 @@ data:
         - action: delete
           key: fluent.tag
       batch:
+        send_batch_max_size: 2048
         send_batch_size: 1024
         timeout: 1s
       filter/exclude_kubelet:

diff --git a/tests/helm/metadata_metrics_otc/static/additional_endpoints.output.yaml b/tests/helm/metadata_metrics_otc/static/additional_endpoints.output.yaml
@@ -104,6 +104,7 @@ data:
       pprof: {}
     processors:
       batch:
+        send_batch_max_size: 2048
         send_batch_size: 1024
         timeout: 1s
       k8s_tagger:

diff --git a/tests/helm/metadata_metrics_otc/static/basic.output.yaml b/tests/helm/metadata_metrics_otc/static/basic.output.yaml
@@ -104,6 +104,7 @@ data:
       pprof: {}
     processors:
       batch:
+        send_batch_max_size: 2048
         send_batch_size: 1024
         timeout: 1s
       k8s_tagger:

diff --git a/tests/helm/otelcol-instrumentation-config/static/traces-gateway-disabled.output.yaml b/tests/helm/otelcol-instrumentation-config/static/traces-gateway-disabled.output.yaml
@@ -37,6 +37,7 @@ data:
       pprof: {}
     processors:
       batch:
+        send_batch_max_size: 512
         send_batch_size: 256
         timeout: 5s
       k8s_tagger:

diff --git a/tests/helm/otelcol-instrumentation-config/static/traces-gateway-enabled.output.yaml b/tests/helm/otelcol-instrumentation-config/static/traces-gateway-enabled.output.yaml
@@ -37,6 +37,7 @@ data:
       pprof: {}
     processors:
       batch:
+        send_batch_max_size: 512
         send_batch_size: 256
         timeout: 5s
       k8s_tagger:

diff --git a/tests/helm/traces-gateway-loadbalancing/static/traces-gateway-true.output.yaml b/tests/helm/traces-gateway-loadbalancing/static/traces-gateway-true.output.yaml
@@ -29,6 +29,7 @@ data:
       pprof: {}
     processors:
       batch:
+        send_batch_max_size: 512
         send_batch_size: 256
         timeout: 5s
       memory_limiter: