feat: adjust queueing and batching configurarion for logs, OTC accord…

…ing to testing results
SumoLogic · Aug 19, 2022 · 984218f · 984218f
1 parent 45a1f72
commit 984218f
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,12 +15,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - chore: upgrade sumologic terraform provider to 2.18 [#2399]
 - chore: update kubernetes-tools to 2.12.0 [#2472]
-- feat(otelcol/metrics): adjust metric otelcol configuration [#2474]
+- feat(otelcol/metrics): adjust metric otelcol configuration [#2474], [#2479]
 
 [#2466]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2466
 [#2399]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2399
 [#2472]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2472
 [#2474]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2474
+[#2479]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2479
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.14.1...main
 
 ## [v2.14.1]

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -3979,7 +3979,7 @@ metadata:
           ## Number of spans after which a batch will be sent regardless of time
           send_batch_size: 1_024
           ## Time duration after which a batch will be sent regardless of size
-          timeout: 30s
+          timeout: 1s
         ## Configuration for Kubernetes Processor
         ## ref: https://github.com/SumoLogic/sumologic-otel-collector/tree/main/pkg/processor/k8sprocessor
         k8s_tagger:
@@ -4151,6 +4151,8 @@ metadata:
           sending_queue:
             enabled: true
             persistent_storage_enabled: '{{ .Values.metadata.persistence.enabled }}'
+            num_consumers: 10
+            queue_size: 10_000
         sumologic/systemd:
           log_format: json
           json_logs:
@@ -4169,6 +4171,8 @@ metadata:
           sending_queue:
             enabled: true
             persistent_storage_enabled: '{{ .Values.metadata.persistence.enabled }}'
+            num_consumers: 10
+            queue_size: 10_000
 
       processors:
         ## Common processors
@@ -4192,9 +4196,9 @@ metadata:
         ## The batch processor accepts spans and places them into batches grouped by node and resource
         batch:
           ## Number of spans after which a batch will be sent regardless of time
-          send_batch_size: 256
+          send_batch_size: 1_024
           ## Time duration after which a batch will be sent regardless of size
-          timeout: 5s
+          timeout: 1s
 
         ## Containers related processors
         filter/include_fluent_tag_containers:

diff --git a/tests/helm/metadata_logs_otc/static/basic.output.yaml b/tests/helm/metadata_logs_otc/static/basic.output.yaml
@@ -22,6 +22,8 @@ data:
         sending_queue:
           enabled: true
           persistent_storage_enabled: true
+          num_consumers: 10
+          queue_size: 10000
         source_category: '%{_sourceCategory}'
         source_host: '%{_sourceHost}'
         source_name: '%{_sourceName}'
@@ -35,6 +37,8 @@ data:
         sending_queue:
           enabled: true
           persistent_storage_enabled: true
+          num_consumers: 10
+          queue_size: 10000
         source_category: '%{_sourceCategory}'
         source_host: '%{_sourceHost}'
         source_name: '%{_sourceName}'
@@ -83,8 +87,8 @@ data:
         - action: delete
           key: fluent.tag
       batch:
-        send_batch_size: 256
-        timeout: 5s
+        send_batch_size: 1024
+        timeout: 1s
       filter/exclude_kubelet:
         logs:
           exclude:

diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml
@@ -22,6 +22,8 @@ data:
         sending_queue:
           enabled: true
           persistent_storage_enabled: true
+          num_consumers: 10
+          queue_size: 10000
         source_category: '%{_sourceCategory}'
         source_host: '%{_sourceHost}'
         source_name: '%{_sourceName}'
@@ -35,6 +37,8 @@ data:
         sending_queue:
           enabled: true
           persistent_storage_enabled: true
+          num_consumers: 10
+          queue_size: 10000
         source_category: '%{_sourceCategory}'
         source_host: '%{_sourceHost}'
         source_name: '%{_sourceName}'
@@ -83,8 +87,8 @@ data:
         - action: delete
           key: fluent.tag
       batch:
-        send_batch_size: 256
-        timeout: 5s
+        send_batch_size: 1024
+        timeout: 1s
       filter/exclude_kubelet:
         logs:
           exclude: