diff --git a/lib/fluent/plugin/grok.rb b/lib/fluent/plugin/grok.rb index fb0b77e..5bbb78a 100644 --- a/lib/fluent/plugin/grok.rb +++ b/lib/fluent/plugin/grok.rb @@ -36,6 +36,9 @@ def initialize(plugin, conf) if @plugin.respond_to?(:keep_time_key) @keep_time_key = @plugin.keep_time_key end + if @plugin.respond_to?(:time_format) + @time_format = @plugin.time_format + end end def add_patterns_from_file(path) @@ -48,10 +51,10 @@ def add_patterns_from_file(path) def setup if @plugin.grok_pattern - @parsers[:grok_pattern] = expand_pattern_expression(@plugin.grok_pattern, @conf) + @parsers[:grok_pattern] = expand_pattern_expression_grok_pattern(@plugin.grok_pattern, @conf) else @plugin.grok_confs.each.with_index do |grok_conf, index| - @parsers[grok_conf.name || index] = expand_pattern_expression(grok_conf.pattern, grok_conf) + @parsers[grok_conf.name || index] = expand_pattern_expression_grok_section(grok_conf) end end @parsers.reject! do |key, parser| @@ -64,7 +67,7 @@ def setup private - def expand_pattern_expression(grok_pattern, conf) + def expand_pattern_expression_grok_pattern(grok_pattern, conf) regexp, types = expand_pattern(grok_pattern) $log.info "Expanded the pattern #{grok_pattern} into #{regexp}" _conf = conf.to_h @@ -83,6 +86,37 @@ def expand_pattern_expression(grok_pattern, conf) nil end + def expand_pattern_expression_grok_section(conf) + regexp, types = expand_pattern(conf.pattern) + $log.info "Expanded the pattern #{conf.pattern} into #{regexp}" + _conf = conf.to_h + unless types.empty? + _conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",") + end + if conf["multiline"] || @multiline_mode + _conf["multiline"] = conf["multiline"] || @multiline_mode + end + if conf["keep_time_key"] || @keep_time_key + _conf["keep_time_key"] = conf["keep_time_key"] || @keep_time_key + end + if conf["time_key"] + _conf["time_key"] = conf["time_key"] + end + if conf["time_format"] || @time_format + _conf["time_format"] = conf["time_format"] || @time_format + end + _conf["expression"] = regexp + config = Fluent::Config::Element.new("parse", "", _conf, []) + parser = Fluent::Plugin::RegexpParser.new + parser.configure(config) + parser + rescue GrokPatternNotFoundError => e + raise e + rescue => e + $log.error(error: e) + nil + end + def expand_pattern(pattern) # It's okay to modify in place. no need to expand it more than once. type_map = {} diff --git a/lib/fluent/plugin/parser_grok.rb b/lib/fluent/plugin/parser_grok.rb index b41dd90..b1f31bd 100644 --- a/lib/fluent/plugin/parser_grok.rb +++ b/lib/fluent/plugin/parser_grok.rb @@ -22,6 +22,9 @@ class GrokParser < Parser config_param :name, :string, default: nil desc "The pattern of grok" config_param :pattern, :string + config_param :keep_time_key, :bool, default: false + config_param :time_key, :string, default: "time" + config_param :time_format, :string, default: nil end def initialize diff --git a/test/test_grok_parser.rb b/test/test_grok_parser.rb index ec5d062..13cdaa7 100644 --- a/test/test_grok_parser.rb +++ b/test/test_grok_parser.rb @@ -321,6 +321,34 @@ class GrokParserTest < ::Test::Unit::TestCase end end + sub_test_case "grok section" do + test "complex pattern" do + d = create_driver(%[ + + pattern %{COMBINEDAPACHELOG} + time_key timestamp + time_format %d/%b/%Y:%H:%M:%S %z + + ]) + expected_record = { + "clientip" => "127.0.0.1", + "ident" => "192.168.0.1", + "auth" => "-", + "verb" => "GET", + "request" => "/", + "httpversion" => "1.1", + "response" => "200", + "bytes" => "777", + "referrer" => "\"-\"", + "agent" => "\"Opera/12.0\"" + } + d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record| + assert_equal(expected_record, record) + assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time) + end + end + end + private def create_driver(conf)