From 1b0f350da9c742006447b184b68a9d6713b56bdd Mon Sep 17 00:00:00 2001 From: Matthias Runge Date: Thu, 23 Jul 2020 11:07:03 +0200 Subject: [PATCH 1/6] Add options to limit the send queue length Collectd has the option to limit the send queue lenght for amqp1. This patch adds support for that. For reference, that was added in https://github.com/collectd/collectd/pull/3432/files --- manifests/plugin/amqp1.pp | 27 +++++++++++++--------- spec/classes/collectd_plugin_amqp1_spec.rb | 2 ++ templates/plugin/amqp1.conf.epp | 3 +++ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifests/plugin/amqp1.pp b/manifests/plugin/amqp1.pp index 68afad45a..d1f29e7b3 100644 --- a/manifests/plugin/amqp1.pp +++ b/manifests/plugin/amqp1.pp @@ -47,6 +47,10 @@ # before attempting to reconnect. # Defaults to 1 # +# [*send_queue_limit*] +# Limits the SentQueue to a defined value, helps to keep memory usage low +# when the write target does not respond. +# # [*interval*] # Interval setting for the plugin # Defaults to undef @@ -100,17 +104,18 @@ # preserved, i.e. passed through. # class collectd::plugin::amqp1 ( - Enum['present', 'absent'] $ensure = 'present', - Boolean $manage_package = $collectd::manage_package, - String $transport = 'metrics', - Stdlib::Host $host = 'localhost', - Stdlib::Port $port = 5672, - String $user = 'guest', - String $password = 'guest', - String $address = 'collectd', - Hash $instances = {}, - Optional[Integer] $retry_delay = undef, - Optional[Integer] $interval = undef, + Enum['present', 'absent'] $ensure = 'present', + Boolean $manage_package = $collectd::manage_package, + String $transport = 'metrics', + Stdlib::Host $host = 'localhost', + Stdlib::Port $port = 5672, + String $user = 'guest', + String $password = 'guest', + String $address = 'collectd', + Hash $instances = {}, + Optional[Integer] $retry_delay = undef, + Optional[Integer] $send_queue_limit = undef, + Optional[Integer] $interval = undef, ) { include collectd diff --git a/spec/classes/collectd_plugin_amqp1_spec.rb b/spec/classes/collectd_plugin_amqp1_spec.rb index dd9d8e34d..ae68420ea 100644 --- a/spec/classes/collectd_plugin_amqp1_spec.rb +++ b/spec/classes/collectd_plugin_amqp1_spec.rb @@ -45,6 +45,7 @@ password: 'password', address: 'address', retry_delay: 30, + send_queue_limit: 40, instances: { instance: { format: 'JSON', @@ -68,6 +69,7 @@ it { is_expected.to contain_file('amqp1.load').with(content: %r{Password "password"}) } it { is_expected.to contain_file('amqp1.load').with(content: %r{Address "address"}) } it { is_expected.to contain_file('amqp1.load').with(content: %r{RetryDelay 30}) } + it { is_expected.to contain_file('amqp1.load').with(content: %r{SendQueueLimit 40}) } it { is_expected.to contain_file('amqp1.load').with(content: %r{}) } it { is_expected.to contain_file('amqp1.load').with(content: %r{Format "JSON"}) } it { is_expected.to contain_file('amqp1.load').with(content: %r{PreSettle true}) } diff --git a/templates/plugin/amqp1.conf.epp b/templates/plugin/amqp1.conf.epp index b7fac2d1c..fdf44989b 100644 --- a/templates/plugin/amqp1.conf.epp +++ b/templates/plugin/amqp1.conf.epp @@ -8,6 +8,9 @@ <% unless $::collectd::plugin::amqp1::retry_delay =~ Undef { -%> RetryDelay <%= $::collectd::plugin::amqp1::retry_delay %> <% } -%> +<% unless $::collectd::plugin::amqp1::send_queue_limit =~ Undef { -%> + SendQueueLimit <%= $::collectd::plugin::amqp1::send_queue_limit %> +<% } -%> <% $::collectd::plugin::amqp1::instances.keys.sort.each |$name| { -%> "> <% $::collectd::plugin::amqp1::instances[$name].keys.sort.each |$key| { -%> From 5485f3868c1892a255f2988c893f7497b42c039d Mon Sep 17 00:00:00 2001 From: Matthias Runge Date: Thu, 23 Jul 2020 14:09:22 +0200 Subject: [PATCH 2/6] Fix check errors (single newline) --- examples/plugins/ceph.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/plugins/ceph.pp b/examples/plugins/ceph.pp index 6b2aa034f..08645088d 100644 --- a/examples/plugins/ceph.pp +++ b/examples/plugins/ceph.pp @@ -3,3 +3,4 @@ class { 'collectd::plugin::ceph': osds => ['osd.0', 'osd.1', 'osd.2'], } + From 38a9b74ae7304ceaf13b66fc596fc7442ccc5f50 Mon Sep 17 00:00:00 2001 From: Matthias Runge Date: Fri, 31 Jul 2020 10:09:35 +0200 Subject: [PATCH 3/6] Revert "Fix check errors (single newline)" This reverts commit 9f098b811d9d1f2d91f83188dd8b24062a65ba98. It is only indirectly connected with the SendQueueLength patch. --- examples/plugins/ceph.pp | 1 + examples/plugins/filecount.pp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/plugins/ceph.pp b/examples/plugins/ceph.pp index 08645088d..2e9cb379a 100644 --- a/examples/plugins/ceph.pp +++ b/examples/plugins/ceph.pp @@ -4,3 +4,4 @@ osds => ['osd.0', 'osd.1', 'osd.2'], } + diff --git a/examples/plugins/filecount.pp b/examples/plugins/filecount.pp index 9b75b132b..993f426d0 100644 --- a/examples/plugins/filecount.pp +++ b/examples/plugins/filecount.pp @@ -7,11 +7,11 @@ }, } -collectd::plugin::filecount::directory { 'foodir': +collectd::plugin::filecount::directory {'foodir': path => '/path/to/dir', } -collectd::plugin::filecount::directory { 'aborted-uploads': +collectd::plugin::filecount::directory {'aborted-uploads': path => '/var/spool/foo/upload', pattern => '.part.*', mtime => '5m', From 88a26d47e36a5c7712ef6f6e136cb4f35953abfb Mon Sep 17 00:00:00 2001 From: Matthias Runge Date: Wed, 5 Aug 2020 11:25:40 +0200 Subject: [PATCH 4/6] Change SendQueueLimit to positive integer including 0. The parameter should not become negative. --- manifests/plugin/amqp1.pp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/manifests/plugin/amqp1.pp b/manifests/plugin/amqp1.pp index d1f29e7b3..ac761443c 100644 --- a/manifests/plugin/amqp1.pp +++ b/manifests/plugin/amqp1.pp @@ -104,18 +104,18 @@ # preserved, i.e. passed through. # class collectd::plugin::amqp1 ( - Enum['present', 'absent'] $ensure = 'present', - Boolean $manage_package = $collectd::manage_package, - String $transport = 'metrics', - Stdlib::Host $host = 'localhost', - Stdlib::Port $port = 5672, - String $user = 'guest', - String $password = 'guest', - String $address = 'collectd', - Hash $instances = {}, - Optional[Integer] $retry_delay = undef, - Optional[Integer] $send_queue_limit = undef, - Optional[Integer] $interval = undef, + Enum['present', 'absent'] $ensure = 'present', + Boolean $manage_package = $collectd::manage_package, + String $transport = 'metrics', + Stdlib::Host $host = 'localhost', + Stdlib::Port $port = 5672, + String $user = 'guest', + String $password = 'guest', + String $address = 'collectd', + Hash $instances = {}, + Optional[Integer] $retry_delay = undef, + Optional[Integer[0]] $send_queue_limit = undef, + Optional[Integer] $interval = undef, ) { include collectd From 8a9044adff2bd57d3c51df9ee2ef2651cd20dfde Mon Sep 17 00:00:00 2001 From: Matthias Runge Date: Wed, 5 Aug 2020 11:28:58 +0200 Subject: [PATCH 5/6] Delete spurious empty lines in ceph example --- examples/plugins/ceph.pp | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/plugins/ceph.pp b/examples/plugins/ceph.pp index 2e9cb379a..6b2aa034f 100644 --- a/examples/plugins/ceph.pp +++ b/examples/plugins/ceph.pp @@ -3,5 +3,3 @@ class { 'collectd::plugin::ceph': osds => ['osd.0', 'osd.1', 'osd.2'], } - - From 7def97b8e63b02b5803337dc922d2bf2681199f2 Mon Sep 17 00:00:00 2001 From: Matthias Runge Date: Wed, 5 Aug 2020 11:59:00 +0200 Subject: [PATCH 6/6] Re-add whitespace to filecount example --- examples/plugins/filecount.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/plugins/filecount.pp b/examples/plugins/filecount.pp index 993f426d0..9b75b132b 100644 --- a/examples/plugins/filecount.pp +++ b/examples/plugins/filecount.pp @@ -7,11 +7,11 @@ }, } -collectd::plugin::filecount::directory {'foodir': +collectd::plugin::filecount::directory { 'foodir': path => '/path/to/dir', } -collectd::plugin::filecount::directory {'aborted-uploads': +collectd::plugin::filecount::directory { 'aborted-uploads': path => '/var/spool/foo/upload', pattern => '.part.*', mtime => '5m',