Skip to content

Commit

Permalink
Bug 1780698: Fix disk pressure for fluentd buffer chunks
Browse files Browse the repository at this point in the history 
This change limits the fluentd queue length to match the number of
chunks on the filesystem.
  • Loading branch information
@periklis
periklis committed Apr 17, 2020
1 parent 5060759 commit 8ceb621
Show file tree
Hide file tree
Showing 6 changed files with 23 additions and 23 deletions.
18 changes: 9 additions & 9 deletions pkg/generators/forwarding/fluentd/fluent_conf_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ var _ = Describe("Generating fluentd config", func() {
<buffer>
@type file
path '/var/lib/fluentd/secureforward_receiver'
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
flush_interval "#{ENV['FORWARD_FLUSH_INTERVAL'] || '5s'}"
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
Expand Down Expand Up @@ -999,7 +999,7 @@ var _ = Describe("Generating fluentd config", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -1043,7 +1043,7 @@ var _ = Describe("Generating fluentd config", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -1088,7 +1088,7 @@ var _ = Describe("Generating fluentd config", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -1132,7 +1132,7 @@ var _ = Describe("Generating fluentd config", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -1177,7 +1177,7 @@ var _ = Describe("Generating fluentd config", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -1221,7 +1221,7 @@ var _ = Describe("Generating fluentd config", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -1266,7 +1266,7 @@ var _ = Describe("Generating fluentd config", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -1310,7 +1310,7 @@ var _ = Describe("Generating fluentd config", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down
8 changes: 4 additions & 4 deletions pkg/generators/forwarding/fluentd/output_conf_es_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ var _ = Describe("Generating fluentd config blocks", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -145,7 +145,7 @@ var _ = Describe("Generating fluentd config blocks", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -203,7 +203,7 @@ var _ = Describe("Generating fluentd config blocks", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -243,7 +243,7 @@ var _ = Describe("Generating fluentd config blocks", func() {
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down
4 changes: 2 additions & 2 deletions pkg/generators/forwarding/fluentd/output_conf_forward_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ var _ = Describe("Generating fluentd secure forward output store config blocks",
<buffer>
@type file
path '/var/lib/fluentd/secureforward_receiver'
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
flush_interval "#{ENV['FORWARD_FLUSH_INTERVAL'] || '5s'}"
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
Expand Down Expand Up @@ -102,7 +102,7 @@ var _ = Describe("Generating fluentd secure forward output store config blocks",
<buffer>
@type file
path '/var/lib/fluentd/secureforward_receiver'
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
flush_interval "#{ENV['FORWARD_FLUSH_INTERVAL'] || '5s'}"
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
Expand Down
8 changes: 4 additions & 4 deletions pkg/generators/forwarding/fluentd/output_conf_syslog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ var _ = Describe("Generating external syslog server output store config blocks",
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -169,7 +169,7 @@ var _ = Describe("Generating external syslog server output store config blocks",
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -207,7 +207,7 @@ var _ = Describe("Generating external syslog server output store config blocks",
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -239,7 +239,7 @@ var _ = Describe("Generating external syslog server output store config blocks",
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down
6 changes: 3 additions & 3 deletions pkg/generators/forwarding/fluentd/templates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ tls_cert_path {{ .SecretPath "ca-bundle.crt"}}
<buffer>
@type file
path '{{.BufferPath}}'
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
flush_interval "#{ENV['FORWARD_FLUSH_INTERVAL'] || '5s'}"
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
Expand Down Expand Up @@ -609,7 +609,7 @@ const storeElasticsearchTemplate = `{{ define "storeElasticsearch" -}}
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down Expand Up @@ -662,7 +662,7 @@ const storeSyslogTemplate = `{{- define "storeSyslog" -}}
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
retry_max_interval "#{ENV['ES_RETRY_WAIT'] || '300'}"
retry_forever true
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '32' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '8m' }"
overflow_action "#{ENV['BUFFER_QUEUE_FULL_ACTION'] || 'block'}"
</buffer>
Expand Down
2 changes: 1 addition & 1 deletion test/files/secure-forward.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<buffer>
@type file
path '/var/lib/fluentd/secureforwardlegacy'
queued_chunks_limit_size "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
queue_limit_length "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
chunk_limit_size "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
flush_interval "#{ENV['FORWARD_FLUSH_INTERVAL'] || '5s'}"
flush_at_shutdown "#{ENV['FLUSH_AT_SHUTDOWN'] || 'false'}"
Expand Down

0 comments on commit 8ceb621

Please sign in to comment.