time_format is ignored in <grok> section #58
Description
Here is a sample fluentd configuration:
<system>
log_level warn
</system>
<source>
@type exec
run_interval 3s
format json
command echo '{"message":"127.0.0.1 - - [21/Nov/2024:17:42:53 +0000] "GET / HTTP/1.1" 200 3189 "-" "check_http/v2.0.x (monitoring-plugins 2.0.x)"}'
<parse>
@type grok
time_key timestamp
<grok>
pattern %{HTTPD_COMBINEDLOG:timestamp:time:%F %T,%L %z}
time_format %d/%b/%Y:%H:%M:%S %z
</grok>
</parse>
tag first
</source>
<filter first>
@type record_transformer
enable_ruby true
<record>
hostname "#{Socket.gethostname}"
time ${time}
</record>
</filter>
<match **>
@type stdout
</match>
OUTPUT
2018-11-23 19:18:29 +0000 [warn]: parameter 'time_format' in <grok>
pattern "(?<timestamp>(?:(?<clientip>(?:(?:(?:(?:((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?)|(?:(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9]))))|(?:\\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)))) (?<ident>(?:(?:[a-zA-Z][a-zA-Z0-9_.+-=:]+)@(?:\\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)))|(?:(?:[a-zA-Z0-9._-]+))) (?<auth>(?:(?:[a-zA-Z][a-zA-Z0-9_.+-=:]+)@(?:\\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)))|(?:(?:[a-zA-Z0-9._-]+))) \\[(?<timestamp>(?:(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]))/(?:\\b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\\b)/(?:(?>\\d\\d){1,2}):(?:(?!<[0-9])(?:(?:2[0123]|[01]?[0-9])):(?:(?:[0-5][0-9]))(?::(?:(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)))(?![0-9])) (?:(?:[+-]?(?:[0-9]+))))\\] \"(?:(?<verb>\\b\\w+\\b) (?<request>\\S+)(?: HTTP/(?<httpversion>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))))))?|(?<rawrequest>.*?))\" (?<response>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))))) (?:(?<bytes>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+))))))|-)) (?<referrer>(?:(?>(?<!\\\\)(?>\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>\'(?>\\\\.|[^\\\\\']+)+\')|\'\'|(?>`(?>\\\\.|[^\\\\`]+)+`)|``)))) (?<agent>(?:(?>(?<!\\\\)(?>\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>\'(?>\\\\.|[^\\\\\']+)+\')|\'\'|(?>`(?>\\\\.|[^\\\\`]+)+`)|``)))))"
time_format %d/%b/%Y:%H:%M:%S %z
</grok> is not used.
2018-11-23 19:18:32.607140903 +0000 first: {"timestamp":"21/Nov/2024:17:42:53 +0000","clientip":"127.0.0.1","ident":"-","auth":"-","verb":"GET","request":"/","httpversion":"1.1","response":"200","bytes":"3189","referrer":"\"-\"","agent":"\"check_http/v2.0.x (monitoring-plugins 2.0.x)\"","hostname":"localhost","time":"2018-11-23 19:18:32 +0000"}
2018-11-23 19:18:35.606768480 +0000 first: {"timestamp":"21/Nov/2024:17:42:53 +0000","clientip":"127.0.0.1","ident":"-","auth":"-","verb":"GET","request":"/","httpversion":"1.1","response":"200","bytes":"3189","referrer":"\"-\"","agent":"\"check_http/v2.0.x (monitoring-plugins 2.0.x)\"","hostname":"localhost","time":"2018-11-23 19:18:35 +0000"}
This one works, but you can't use multiply patterns obviously:
<system>
log_level warn
</system>
<source>
@type exec
run_interval 3s
format json
command echo '{"message":"127.0.0.1 - - [21/Nov/2024:17:42:53 +0000] "GET / HTTP/1.1" 200 3189 "-" "check_http/v2.0.x (monitoring-plugins 2.0.x)"}'
<parse>
@type grok
time_format %d/%b/%Y:%H:%M:%S %z
time_key timestamp
grok_pattern %{HTTPD_COMBINEDLOG:timestamp:time:%F %T,%L %z}
</parse>
tag first
</source>
<filter first>
@type record_transformer
enable_ruby true
<record>
hostname "#{Socket.gethostname}"
time ${time}
</record>
</filter>
<match **>
@type stdout
</match>
OUTPUT
2024-11-21 17:42:53.000000000 +0000 first: {"clientip":"127.0.0.1","ident":"-","auth":"-","verb":"GET","request":"/","httpversion":"1.1","response":"200","bytes":"3189","referrer":"\"-\"","agent":"\"check_http/v2.0.x (monitoring-plugins 2.0.x)\"","hostname":"localhost","time":"2024-11-21 17:42:53 +0000"}
2024-11-21 17:42:53.000000000 +0000 first: {"clientip":"127.0.0.1","ident":"-","auth":"-","verb":"GET","request":"/","httpversion":"1.1","response":"200","bytes":"3189","referrer":"\"-\"","agent":"\"check_http/v2.0.x (monitoring-plugins 2.0.x)\"","hostname":"localhost","time":"2024-11-21 17:42:53 +0000"}