Add grok_name_key

.travis.yml

@@ -2,8 +2,8 @@ sudo: false
 language: ruby
 
 rvm:
-  - 2.1
-  - 2.2
-  - 2.3.3
-  - 2.4.0
+  - 2.2.10
+  - 2.3.7
+  - 2.4.4
+  - 2.5.1
 

README.md

@@ -95,21 +95,19 @@ You can use this parser without `multiline_start_regexp` when you know your data
 
 ## Configurations
 
-**time_format**
+* See also: [TimeParameters Plugin Overview](https://docs.fluentd.org/v1.0/articles/timeparameters-plugin-overview)
+* See also: [Parser Plugin Overview](https://docs.fluentd.org/v1.0/articles/parser-plugin-overview)
 
-The format of the time field.
+* **time_format** (string) (optional): The format of the time field.
+* **grok_pattern** (string) (optional): The pattern of grok. You cannot specify multiple grok pattern with this.
+* **custom_pattern_path** (string) (optional): Path to the file that includes custom grok patterns
+* **grok_failure_key** (string) (optional): The key has grok failure reason.
+* **grok_name_key** (string) (optional): The key name to store grok section's name
+* **multi_line_start_regexp** (string) (optional): The regexp to match beginning of multiline. This is only for "multiline_grok".
 
-**grok_pattern**
+## Examples
 
-The pattern of grok. You cannot specify multiple grok pattern with this.
-
-**custom_pattern_path**
-
-Path to the file that includes custom grok patterns
-
-**grok_failure_key**
-
-The key has grok failure reason. Default is `nil`.
+### Using grok\_failure\_key
 
 ```aconf
 <source>
@@ -149,21 +147,42 @@ This generates following events:
 2016-11-28 13:07:09.010400923 +0900 dummy.log: {"message1":"/","prog":"bar","path":"/"}
 ```
 
-
-**grok/pattern**
-
-Section for grok patterns. You can use multiple grok patterns with
-multiple `<grok>` sections.
+### Using grok\_name\_key
 
 ```aconf
-<grok>
-  pattern %{IP:ipaddress}
-</grok>
+<source>
+  @type tail
+  path /path/to/log
+  tag grokked_log
+  grok_name_key grok_name
+  grok_failure_key grokfailure
+  <parse>
+    @type grok
+    <grok>
+      name apache_log
+      pattern %{COMBINEDAPACHELOG}
+      time_format "%d/%b/%Y:%H:%M:%S %z"
+    </grok>
+    <grok>
+      name ip_address
+      pattern %{IP:ip_address}
+    </grok>
+    <grok>
+      name rest_message
+      pattern %{GREEDYDATA:message}
+    </grok>
+  </parse>
+</source>
 ```
 
-**multiline_start_regexp**
+This will add keys like following:
+
+* Add `grok_name: "apache_log"` if the record matches `COMBINEDAPACHELOG`
+* Add `grok_name: "ip_address"` if the record matches `IP`
+* Add `grok_name: "rest_message"` if the record matches `GREEDYDATA`
 
-The regexp to match beginning of multiline. This is only for "multiline_grok".
+Add `grokfailure` key to the record if the record does not match any grok pattern.
+See also test code for more details.
 
 ## How to write Grok patterns
 

lib/fluent/plugin/grok.rb

@@ -23,7 +23,7 @@ class GrokPatternNotFoundError < StandardError
 
     def initialize(plugin, conf)
       @pattern_map = {}
-      @parsers = []
+      @parsers = {}
       @multiline_mode = false
       @conf = conf
       @plugin = plugin
@@ -45,13 +45,15 @@ def add_patterns_from_file(path)
 
     def setup
       if @plugin.grok_pattern
-        @parsers << expand_pattern_expression(@plugin.grok_pattern, @conf)
+        @parsers[:grok_pattern] = expand_pattern_expression(@plugin.grok_pattern, @conf)
       else
-        @plugin.grok_confs.each do |grok_conf|
-          @parsers << expand_pattern_expression(grok_conf.pattern, grok_conf)
+        @plugin.grok_confs.each.with_index do |grok_conf, index|
+          @parsers[grok_conf.name || index] = expand_pattern_expression(grok_conf.pattern, grok_conf)
         end
       end
-      @parsers.compact!
+      @parsers.reject! do |key, parser|
+        parser.nil?
+      end
       if @parsers.empty?
         raise Fluent::ConfigError, 'no grok patterns. Check configuration, e.g. typo, configuration syntax, etc'
       end

lib/fluent/plugin/parser_grok.rb

@@ -14,8 +14,12 @@ class GrokParser < Parser
       config_param :custom_pattern_path, :string, default: nil
       desc "The key has grok failure reason"
       config_param :grok_failure_key, :string, default: nil
+      desc "The key name to store grok section's name"
+      config_param :grok_name_key, :string, default: nil
 
       config_section :grok, param_name: "grok_confs", multi: true do
+        desc "The name of this grok section"
+        config_param :name, :string, default: nil
         desc "The pattern of grok"
         config_param :pattern, :string
       end
@@ -49,9 +53,10 @@ def configure(conf={})
       end
 
       def parse(text)
-        @grok.parsers.each do |parser|
+        @grok.parsers.each do |name_or_index, parser|
           parser.parse(text) do |time, record|
             if time and record
+              record[@grok_name_key] = name_or_index if @grok_name_key
               yield time, record
               return
             end

lib/fluent/plugin/parser_multiline_grok.rb

@@ -17,9 +17,10 @@ def firstline?(text)
       end
 
       def parse(text)
-        @grok.parsers.each do |parser|
+        @grok.parsers.each do |name_or_index, parser|
           parser.parse(text) do |time, record|
             if time and record
+              record[@grok_name_key] = name_or_index if @grok_name_key
               yield time, record
               return
             end

test/test_grok_parser.rb

@@ -191,6 +191,116 @@ class GrokParserTest < ::Test::Unit::TestCase
     assert_equal("unknown value conversion for key:'path', type:'foo'", error_message)
   end
 
+  sub_test_case "grok_name_key" do
+    test "one grok section with name" do
+      d = create_driver(%[
+        grok_name_key grok_name
+        <grok>
+          name path
+          pattern %{PATH:path}
+        </grok>
+      ])
+      expected = {
+        "path" => "/",
+        "grok_name" => "path"
+      }
+      d.instance.parse("/") do |time, record|
+        assert_equal(expected, record)
+      end
+    end
+
+    test "one grok section without name" do
+      d = create_driver(%[
+        grok_name_key grok_name
+        <grok>
+          pattern %{PATH:path}
+        </grok>
+      ])
+      expected = {
+        "path" => "/",
+        "grok_name" => 0
+      }
+      d.instance.parse("/") do |time, record|
+        assert_equal(expected, record)
+      end
+    end
+
+    test "multiple grok sections with name" do
+      d = create_driver(%[
+        grok_name_key grok_name
+        <grok>
+          name path
+          pattern %{PATH:path}
+        </grok>
+        <grok>
+          name ip
+          pattern %{IP:ip_address}
+        </grok>
+      ])
+      expected = [
+        { "path" => "/", "grok_name" => "path" },
+        { "ip_address" => "127.0.0.1", "grok_name" => "ip" },
+      ]
+      records = []
+      d.instance.parse("/") do |time, record|
+        records << record
+      end
+      d.instance.parse("127.0.0.1") do |time, record|
+        records << record
+      end
+      assert_equal(expected, records)
+    end
+
+    test "multiple grok sections without name" do
+      d = create_driver(%[
+        grok_name_key grok_name
+        <grok>
+          pattern %{PATH:path}
+        </grok>
+        <grok>
+          pattern %{IP:ip_address}
+        </grok>
+      ])
+      expected = [
+        { "path" => "/", "grok_name" => 0 },
+        { "ip_address" => "127.0.0.1", "grok_name" => 1 },
+      ]
+      records = []
+      d.instance.parse("/") do |time, record|
+        records << record
+      end
+      d.instance.parse("127.0.0.1") do |time, record|
+        records << record
+      end
+      assert_equal(expected, records)
+    end
+
+    test "multiple grok sections with both name and index" do
+      d = create_driver(%[
+        grok_name_key grok_name
+        <grok>
+          name path
+          pattern %{PATH:path}
+        </grok>
+        <grok>
+          pattern %{IP:ip_address}
+        </grok>
+      ])
+      expected = [
+        { "path" => "/", "grok_name" => "path" },
+        { "ip_address" => "127.0.0.1", "grok_name" => 1 },
+      ]
+      records = []
+      d.instance.parse("/") do |time, record|
+        records << record
+      end
+      d.instance.parse("127.0.0.1") do |time, record|
+        records << record
+      end
+      assert_equal(expected, records)
+    end
+  end
+
   private
 
   def create_driver(conf)