From 65f12674579e0c9349aaf291f22b785bca084275 Mon Sep 17 00:00:00 2001 From: Ben Chadwick Date: Tue, 19 Nov 2013 15:54:38 -0500 Subject: [PATCH 1/4] JSON filter with array split feature --- lib/logstash/filters/json.rb | 43 +++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/logstash/filters/json.rb b/lib/logstash/filters/json.rb index 569bfe8f949..c46958ed4cd 100644 --- a/lib/logstash/filters/json.rb +++ b/lib/logstash/filters/json.rb @@ -1,6 +1,6 @@ -# encoding: utf-8 require "logstash/filters/base" require "logstash/namespace" +require "logstash/event" # JSON filter. Takes a field that contains JSON and expands it into # an actual datastructure. @@ -41,6 +41,9 @@ class LogStash::Filters::Json < LogStash::Filters::Base # Note: if the "target" field already exists, it will be overwritten. config :target, :validate => :string + # Define a key containing an array of events to be split up and fed into LogStash as separate individual events. + config :array_split, :validate => :string + public def register # Nothing to do here @@ -62,20 +65,34 @@ def filter(event) end begin - # TODO(sissel): Note, this will not successfully handle json lists - # like your text is '[ 1,2,3 ]' JSON.parse gives you an array (correctly) - # which won't merge into a hash. If someone needs this, we can fix it - # later. - dest.merge!(JSON.parse(event[@source])) - - # This is a hack to help folks who are mucking with @timestamp during - # their json filter. You aren't supposed to do anything with "@timestamp" - # outside of the date filter, but nobody listens... ;) - if event["@timestamp"].is_a?(String) - event["@timestamp"] = Time.parse(event["@timestamp"]).gmtime + source = JSON.parse(event[@source]) + + if !!@array_split + + source[@array_split].each do |item| + next if item.empty? + + event_split = LogStash::Event.new(item.clone) + @logger.debug("JSON Array Split item", :value => event_split) + filter_matched(event_split) + + yield event_split # yield each item to be handled individually + end + event.cancel + + else + dest.merge!(JSON.parse(event[@source])) + + # This is a hack to help folks who are mucking with @timestamp during + # their json filter. You aren't supposed to do anything with "@timestamp" + # outside of the date filter, but nobody listens... ;) + if event["@timestamp"].is_a?(String) + event["@timestamp"] = Time.parse(event["@timestamp"]).gmtime + end + + filter_matched(event) end - filter_matched(event) rescue => e event.tag("_jsonparsefailure") @logger.warn("Trouble parsing json", :source => @source, From 8880d6ed2b8d17f7ce0957fbde15b8b82aac21c0 Mon Sep 17 00:00:00 2001 From: Ben Chadwick Date: Wed, 20 Nov 2013 18:31:48 -0500 Subject: [PATCH 2/4] Moved JSON split feature into split plugin --- lib/logstash/filters/json.rb | 43 +++++++++++------------------------ lib/logstash/filters/split.rb | 37 ++++++++++++++++++------------ 2 files changed, 36 insertions(+), 44 deletions(-) diff --git a/lib/logstash/filters/json.rb b/lib/logstash/filters/json.rb index c46958ed4cd..569bfe8f949 100644 --- a/lib/logstash/filters/json.rb +++ b/lib/logstash/filters/json.rb @@ -1,6 +1,6 @@ +# encoding: utf-8 require "logstash/filters/base" require "logstash/namespace" -require "logstash/event" # JSON filter. Takes a field that contains JSON and expands it into # an actual datastructure. @@ -41,9 +41,6 @@ class LogStash::Filters::Json < LogStash::Filters::Base # Note: if the "target" field already exists, it will be overwritten. config :target, :validate => :string - # Define a key containing an array of events to be split up and fed into LogStash as separate individual events. - config :array_split, :validate => :string - public def register # Nothing to do here @@ -65,34 +62,20 @@ def filter(event) end begin - source = JSON.parse(event[@source]) - - if !!@array_split - - source[@array_split].each do |item| - next if item.empty? - - event_split = LogStash::Event.new(item.clone) - @logger.debug("JSON Array Split item", :value => event_split) - filter_matched(event_split) - - yield event_split # yield each item to be handled individually - end - event.cancel - - else - dest.merge!(JSON.parse(event[@source])) - - # This is a hack to help folks who are mucking with @timestamp during - # their json filter. You aren't supposed to do anything with "@timestamp" - # outside of the date filter, but nobody listens... ;) - if event["@timestamp"].is_a?(String) - event["@timestamp"] = Time.parse(event["@timestamp"]).gmtime - end - - filter_matched(event) + # TODO(sissel): Note, this will not successfully handle json lists + # like your text is '[ 1,2,3 ]' JSON.parse gives you an array (correctly) + # which won't merge into a hash. If someone needs this, we can fix it + # later. + dest.merge!(JSON.parse(event[@source])) + + # This is a hack to help folks who are mucking with @timestamp during + # their json filter. You aren't supposed to do anything with "@timestamp" + # outside of the date filter, but nobody listens... ;) + if event["@timestamp"].is_a?(String) + event["@timestamp"] = Time.parse(event["@timestamp"]).gmtime end + filter_matched(event) rescue => e event.tag("_jsonparsefailure") @logger.warn("Trouble parsing json", :source => @source, diff --git a/lib/logstash/filters/split.rb b/lib/logstash/filters/split.rb index 3524f0d6fa6..31f681bf61e 100644 --- a/lib/logstash/filters/split.rb +++ b/lib/logstash/filters/split.rb @@ -1,6 +1,6 @@ -# encoding: utf-8 require "logstash/filters/base" require "logstash/namespace" +require "logstash/event" # The split filter is for splitting multiline messages into separate events. # @@ -22,6 +22,10 @@ class LogStash::Filters::Split < LogStash::Filters::Base # The field which value is split by the terminator config :field, :validate => :string, :default => "message" + # If true, the array data will be passed forward as a single hash element with @field as the key. + # If false, treat the entire array element as a new event for further processing. + config :reuse_element, :validate => :boolean, :default => true + public def register # Nothing to do @@ -31,27 +35,32 @@ def register def filter(event) return unless filter?(event) - events = [] + splits = [] original_value = event[@field] - # If for some reason the field is an array of values, take the first only. - original_value = original_value.first if original_value.is_a?(Array) - - # Using -1 for 'limit' on String#split makes ruby not drop trailing empty - # splits. - splits = original_value.split(@terminator, -1) + if original_value.is_a?(Array) + splits = original_value + else + # Using -1 for 'limit' on String#split makes ruby not drop trailing empty + # splits. + splits = original_value.split(@terminator, -1) + end - # Skip filtering if splitting this event resulted in only one thing found. - return if splits.length == 1 - #or splits[1].empty? + # Skip filtering if splitting this event resulted in only one thing found + return event if splits.length <= 1 splits.each do |value| next if value.empty? - event_split = event.clone - @logger.debug("Split event", :value => value, :field => @field) - event_split[@field] = value + event_split = nil + if @reuse_element + event_split = event.clone + @logger.debug("Split event", :value => value, :field => @field) + event_split[@field] = value + else + event_split = LogStash::Event.new(value) + end filter_matched(event_split) # Push this new event onto the stack at the LogStash::FilterWorker From d954984df4f9dad0e86ac21108efe3283b394b41 Mon Sep 17 00:00:00 2001 From: Ben Chadwick Date: Fri, 22 Nov 2013 13:27:31 -0500 Subject: [PATCH 3/4] fixed return in split filter --- lib/logstash/filters/split.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/filters/split.rb b/lib/logstash/filters/split.rb index 31f681bf61e..352f18ead52 100644 --- a/lib/logstash/filters/split.rb +++ b/lib/logstash/filters/split.rb @@ -48,7 +48,7 @@ def filter(event) end # Skip filtering if splitting this event resulted in only one thing found - return event if splits.length <= 1 + return if splits.length <= 1 splits.each do |value| next if value.empty? From 71f6354a522fcc5ed6e7d6556c250d076f757ed8 Mon Sep 17 00:00:00 2001 From: Ben Chadwick Date: Fri, 22 Nov 2013 13:51:52 -0500 Subject: [PATCH 4/4] Removed needless declaration --- lib/logstash/filters/split.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/logstash/filters/split.rb b/lib/logstash/filters/split.rb index 352f18ead52..8e52a010fef 100644 --- a/lib/logstash/filters/split.rb +++ b/lib/logstash/filters/split.rb @@ -53,7 +53,6 @@ def filter(event) splits.each do |value| next if value.empty? - event_split = nil if @reuse_element event_split = event.clone @logger.debug("Split event", :value => value, :field => @field)