shipper queue size configuration

elastic · Jan 7, 2016 · 6865d1a · 6865d1a
1 parent acad1cd
commit 6865d1a
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 0 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -28,6 +28,7 @@ https://github.com/elastic/beats/compare/1.0.0...master[Check the HEAD diff]
 *Affecting all Beats*
 - Fix logging issue with file based output where newlines could be misplaced
   during concurrent logging {pull}650[650]
+- Reduce memory usage by separate queue sizes for single events and bulk events. {pull}649[649] {issue}516[516]
 
 *Packetbeat*
 - Fix setting direction to out and use its value to decide when dropping events if ignore_outgoing is enabled {pull}557[557]

diff --git a/filebeat/etc/filebeat.yml b/filebeat/etc/filebeat.yml
@@ -345,6 +345,9 @@ shipper:
   # refresh_topology_freq. The default is 15 seconds.
   #topology_expire: 15
 
+  # Internal queue size for single events in processing pipeline
+  #queue_size: 1000
+
   # Configure local GeoIP database support.
   # If no paths are not configured geoip is disabled.
   #geoip:

diff --git a/libbeat/docs/shipperconfig.asciidoc b/libbeat/docs/shipperconfig.asciidoc
@@ -51,6 +51,7 @@ shipper:
     #paths:
     #  - "/usr/share/GeoIP/GeoLiteCity.dat"
     #  - "/usr/local/var/GeoIP/GeoLiteCity.dat"
+
 ------------------------------------------------------------------------------
 
 ==== Options
@@ -143,6 +144,15 @@ useful in case a Beat stops publishing its IP addresses. The IP addresses
 are removed automatically from the topology map after expiration. The default
 is 15 seconds.
 
+===== queue_size
+
+Configure internal queue sizes for single events in processing pipeline. Default
+value is 1000.
+
+===== bulk_queue_size
+
+(DO NOT TOUCH) Configure internal queue size for bulk events in processing pipeline. Default value is 0.
+
 ===== geoip.paths
 
 This configuration option is currently used by Packetbeat only.

diff --git a/libbeat/etc/libbeat.yml b/libbeat/etc/libbeat.yml
@@ -183,6 +183,9 @@ shipper:
   # refresh_topology_freq. The default is 15 seconds.
   #topology_expire: 15
 
+  # Internal queue size for single events in processing pipeline
+  #queue_size: 1000
+
   # Configure local GeoIP database support.
   # If no paths are not configured geoip is disabled.
   #geoip:

diff --git a/packetbeat/etc/packetbeat.yml b/packetbeat/etc/packetbeat.yml
@@ -322,6 +322,9 @@ shipper:
   # refresh_topology_freq. The default is 15 seconds.
   #topology_expire: 15
 
+  # Internal queue size for single events in processing pipeline
+  #queue_size: 1000
+
   # Configure local GeoIP database support.
   # If no paths are not configured geoip is disabled.
   #geoip:

diff --git a/topbeat/etc/topbeat.yml b/topbeat/etc/topbeat.yml
@@ -209,6 +209,9 @@ shipper:
   # refresh_topology_freq. The default is 15 seconds.
   #topology_expire: 15
 
+  # Internal queue size for single events in processing pipeline
+  #queue_size: 1000
+
   # Configure local GeoIP database support.
   # If no paths are not configured geoip is disabled.
   #geoip:

diff --git a/winlogbeat/etc/winlogbeat.yml b/winlogbeat/etc/winlogbeat.yml
@@ -208,6 +208,9 @@ shipper:
   # refresh_topology_freq. The default is 15 seconds.
   #topology_expire: 15
 
+  # Internal queue size for single events in processing pipeline
+  #queue_size: 1000
+
   # Configure local GeoIP database support.
   # If no paths are not configured geoip is disabled.
   #geoip: