fluentd eat disk

[zh7606559342012]

Describe the bug

fluentd eat disk, systemctl restart td-agent can free the eat disk

To Reproduce

Expected behavior

Your Environment

- Fluentd version:0.0.0
- TD Agent version:4.4.2
- Operating system:centos7.9
- Kernel version:3.10

Your Configuration

####
## Output descriptions:
##

# Treasure Data (http://www.treasure-data.com/) provides cloud based data
# analytics platform, which easily stores and processes data from td-agent.
# FREE plan is also provided.
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE
<match td.*.*>
  @type tdlog
  @id output_td
  apikey YOUR_API_KEY

  auto_create_table
  <buffer>
    @type file
    path /var/log/td-agent/buffer/td
  </buffer>

  <secondary>
    @type file
    path /var/log/td-agent/failed_records
  </secondary>
</match>

## match tag=debug.** and dump to console
<match debug.**>
  @type stdout
  @id output_stdout
</match>

####
## Source descriptions:
##

## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
#<source>
#  @type forward
#  @id input_forward
#</source>

## built-in UNIX socket input
#<source>
#  type unix
#</source>

# HTTP input
# POST http://localhost:8888/<tag>?json=<json>
# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
# @see http://docs.fluentd.org/articles/in_http
#<source>
#  @type http
#  @id input_http
#  port 8888
#</source>

## live debugging agent
<source>
  @type debug_agent
  @id input_debug_agent
  bind 127.0.0.1
  port 24230
</source>

####
## Examples:
##

## File input
## read apache logs continuously and tags td.apache.access
#<source>
#  @type tail
#  @id input_tail
#  <parse>
#    @type apache2
#  </parse>
#  path /var/log/httpd-access.log
#  tag td.apache.access
#</source>

## File output
## match tag=local.** and write to file
#<match local.**>
#  @type file
#  @id output_file
#  path /var/log/td-agent/access
#</match>

## Forwarding
## match tag=system.** and forward to another td-agent server
#<match system.**>
#  @type forward
#  @id output_system_forward
#
#  <server>
#    host 192.168.0.11
#  </server>
#  # secondary host is optional
#  <secondary>
#    <server>
#      host 192.168.0.12
#    </server>
#  </secondary>
#</match>

## Multiple output
## match tag=td.*.* and output to Treasure Data AND file
#<match td.*.*>
#  @type copy
#  @id output_copy
#  <store>
#    @type tdlog
#    apikey API_KEY
#    auto_create_table
#    <buffer>
#      @type file
#      path /var/log/td-agent/buffer/td
#    </buffer>
#  </store>
#  <store>
#    @type file
#    path /var/log/td-agent/td-%Y-%m-%d/%H.log
#  </store>
#</match>

<source>
  @type tail
  @id cert_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/cert_fail.log.pos
  tag cert_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter cert_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /no issuer certificate found for/
  </regexp>
</filter>
<match cert_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id psk_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/psk_fail.log.pos
  tag psk_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter psk_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /shared key for/
  </regexp>
</filter>
<match psk_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id rsa_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/rsa_fail.log.pos
  tag rsa_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter rsa_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /no matching proposal found, trying alternative config/
  </regexp>
</filter>
<match rsa_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id whitelist_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/whitelist_fail.log.pos
  tag whitelist_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter whitelist_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /not whitelisted/
  </regexp>
</filter>
<match whitelist_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id eap_aka_undo_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/eap_aka_undo_fail.log.pos
  tag eap_aka_undo_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter eap_aka_undo_parse>
  @type grep
  <regexp>
    key content
    pattern /failed to map pseudonym/
  </regexp>
</filter>
<match eap_aka_undo_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id eap_aka_done_err_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/eap_aka_done_err_fail.log.pos
  tag eap_aka_done_err_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter eap_aka_done_err_parse>
  @type grep
  <regexp>
    key content
    pattern /received AKA_AUTHENTICATION_REJECT, authentication failed/
  </regexp>
</filter>
<match eap_aka_done_err_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id child_sa_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/child_sa_fail.log.pos
  tag child_sa_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter child_sa_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /CHILD_SA rekeying failed/
  </regexp>
</filter>
<match child_sa_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/tdalarm
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id datagw_alarm_parse
  path /var/log/datagw/datagw.log
  pos_file /var/log/td-agent/datagw_alarm.log.pos
  tag datagw_alarm_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^:]+:.*?) ([^:]+:.*?)(?<content>.+)$
  </parse>
</source>
<filter datagw_alarm_parse>
  @type grep
  <regexp>
    key content
    pattern /t receive arp reply, generate alarm/
  </regexp>
</filter>
<match datagw_alarm_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/tdalarm
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id datagw_alarm_recv_parse
  path /var/log/datagw/datagw.log
  pos_file /var/log/td-agent/datagw_alarm_recv.log.pos
  tag datagw_alarm_recv_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^:]+:.*?) ([^:]+:.*?)(?<content>.+)$
  </parse>
</source>
<filter datagw_alarm_recv_parse>
  @type grep
  <regexp>
    key content
    pattern /received arp reply, clear alarm/
  </regexp>
</filter>
<match datagw_alarm_recv_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/tdalarm
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

Your Error Log

none

Additional context

No response

[zh7606559342012]

i saw the question #480

the same question i meet now, i wonder if it was caused by the logrotate.d conf

cat << EOF > /etc/logrotate.d/td-agent
/var/log/td-agent/td-agent.log {
missingok
daily
minsize 10M
create 0600 root
rotate 7
notifempty
compress
postrotate
systemctl restart rsyslog
endscript
}
EOF

need add copytruncate can avoid the question?
thanks for reply!
Astin ^_^