Merge pull request #56 from dwbutler/formatters

Support for different formatters

.travis.yml

@@ -14,3 +14,4 @@ gemfile:
 matrix:
   allow_failures:
     - rbx-2
+sudo: false

README.md

@@ -28,7 +28,7 @@ Or install it yourself as:
 
     $ gem install logstash-logger
 
-## Basic Usage
+## Usage Examples
 
 ```ruby
 require 'logstash-logger'
@@ -49,6 +49,12 @@ stdout_logger = LogStashLogger.new(type: :stdout)
 stderr_logger = LogStashLogger.new(type: :stderr)
 io_logger = LogStashLogger.new(type: :io, io: io)
 
+# Use a different formatter
+cee_logger = LogStashLogger.new(type: :tcp, host: 'logsene-receiver-syslog.sematext.com', port: 514, formatter: :cee_syslog)
+custom_formatted_logger = LogStashLogger.new(type: :redis, formatter: MyCustomFormatter)
+lambda_formatted_logger = LogStashLogger.new(type: :stdout, formatter: ->(severity, time, progname, msg) { "[#{progname}] #{msg}" })
+ruby_default_formatter_logger = LogStashLogger.new(type: :file, path: 'log/development.log', formatter: ::Logger::Formatter)
+
 # Multiple Outputs
 multi_logger = LogStashLogger.new([{type: :file, path: 'log/development.log'}, {type: :udp, host: 'localhost', port: 5228}])
 
@@ -77,7 +83,7 @@ logger.tagged('foo') { logger.fatal('bar') }
 ## URI Configuration
 You can use a URI to configure your logstash logger instead of a hash. This is useful in environments
 such as Heroku where you may want to read configuration values from the environment. The URI scheme
-is `type://host:port/path`. Some sample URI configurations are given below.
+is `type://host:port/path?key=value`. Some sample URI configurations are given below.
 
 ```
 udp://localhost:5228
@@ -97,7 +103,7 @@ Pass the URI into your logstash logger like so:
 logger = LogStashLogger.new(uri: ENV['LOGSTASH_URI'])
 ```
 
-## Logstash Configuration
+## Logstash Listener Configuration
 
 In order for logstash to correctly receive and parse the events, you will need to
 configure and run a listener that uses the `json_lines` codec. For example, to receive

lib/logstash-logger/formatter.rb

@@ -1,51 +1,48 @@
-require 'logger'
-require 'socket'
-require 'time'
+require 'logstash-logger/formatter/base'
 
 module LogStashLogger
-  HOST = ::Socket.gethostname
+	module Formatter
+		DEFAULT_FORMATTER = :json_lines
 
-  class Formatter < ::Logger::Formatter
-    include TaggedLogging::Formatter
+    autoload :LogStashEvent, 'logstash-logger/formatter/logstash_event'
+    autoload :Json, 'logstash-logger/formatter/json'
+    autoload :JsonLines, 'logstash-logger/formatter/json_lines'
+    autoload :Cee, 'logstash-logger/formatter/cee'
+    autoload :CeeSyslog, 'logstash-logger/formatter/cee_syslog'
 
-    def call(severity, time, progname, message)
-      event = build_event(message, severity, time)
-      "#{event.to_json}\n"
+		def self.new(formatter_type)
+    	build_formatter(formatter_type)
     end
 
-    protected
+    def self.build_formatter(formatter_type)
+      formatter_type ||= DEFAULT_FORMATTER
 
-    def build_event(message, severity, time)
-      data = message
-      if data.is_a?(String) && data.start_with?('{')
-        data = (JSON.parse(message) rescue nil) || message
+      if custom_formatter_instance?(formatter_type)
+        formatter_type
+      elsif custom_formatter_class?(formatter_type)
+        formatter_type.new
+      else
+        formatter_klass(formatter_type).new
       end
+    end
 
-      event = case data
-                when LogStash::Event
-                  data.clone
-                when Hash
-                  event_data = data.merge("@timestamp" => time)
-                  LogStash::Event.new(event_data)
-                else
-                  LogStash::Event.new("message" => msg2str(data), "@timestamp" => time)
-              end
-
-      event['severity'] ||= severity
-      #event.type = progname
-
-      event['host'] ||= HOST
+    def self.formatter_klass(formatter_type)
+      case formatter_type.to_sym
+      when :json_lines then JsonLines
+      when :json then Json
+      when :logstash_event then LogStashEvent
+      when :cee then Cee
+      when :cee_syslog then CeeSyslog
+      else fail ArgumentError, 'Invalid formatter'
+      end
+    end
 
-      current_tags.each { |tag| event.tag(tag) }
-
-      LogStashLogger.configuration.customize_event_block.call(event) if LogStashLogger.configuration.customize_event_block.respond_to?(:call)
+    def self.custom_formatter_instance?(formatter_type)
+      formatter_type.respond_to?(:call)
+    end
 
-      # In case Time#to_json has been overridden
-      if event.timestamp.is_a?(Time)
-        event.timestamp = event.timestamp.iso8601(3)
-      end
-
-      event
+    def self.custom_formatter_class?(formatter_type)
+      formatter_type.is_a?(Class) && formatter_type.method_defined?(:call)
     end
-  end
+	end
 end

lib/logstash-logger/formatter/base.rb

@@ -0,0 +1,52 @@
+require 'logger'
+require 'socket'
+require 'time'
+
+module LogStashLogger
+  module Formatter
+    HOST = ::Socket.gethostname
+
+    class Base < ::Logger::Formatter
+      include ::LogStashLogger::TaggedLogging::Formatter
+
+      def call(severity, time, progname, message)
+        @event = build_event(message, severity, time)
+      end
+
+      protected
+
+      def build_event(message, severity, time)
+        data = message
+        if data.is_a?(String) && data.start_with?('{'.freeze)
+          data = (JSON.parse(message) rescue nil) || message
+        end
+
+        event = case data
+                  when LogStash::Event
+                    data.clone
+                  when Hash
+                    event_data = data.merge("@timestamp".freeze => time)
+                    LogStash::Event.new(event_data)
+                  else
+                    LogStash::Event.new("message".freeze => msg2str(data), "@timestamp".freeze => time)
+                end
+
+        event['severity'.freeze] ||= severity
+        #event.type = progname
+
+        event['host'.freeze] ||= HOST
+
+        current_tags.each { |tag| event.tag(tag) }
+
+        LogStashLogger.configuration.customize_event_block.call(event) if LogStashLogger.configuration.customize_event_block.respond_to?(:call)
+
+        # In case Time#to_json has been overridden
+        if event.timestamp.is_a?(Time)
+          event.timestamp = event.timestamp.iso8601(3)
+        end
+
+        event
+      end
+    end
+  end
+end

lib/logstash-logger/formatter/cee.rb

@@ -0,0 +1,10 @@
+module LogStashLogger
+  module Formatter
+    class Cee < Base
+      def call(severity, time, progname, message)
+        super
+        "@cee:#{@event.to_json}"
+      end
+    end
+  end
+end

lib/logstash-logger/formatter/cee_syslog.rb

@@ -0,0 +1,20 @@
+module LogStashLogger
+  module Formatter
+    class CeeSyslog < Cee
+      def call(severity, time, progname, message)
+        @cee = super
+        @progname = progname
+
+        "#{facility}:#{@cee}\n"
+      end
+
+      protected
+
+      def facility
+        @facility = "#{@event['host']}"
+        @facility << " #{@progname}" if @progname
+        @facility
+      end
+    end
+  end
+end

lib/logstash-logger/formatter/json.rb

@@ -0,0 +1,10 @@
+module LogStashLogger
+  module Formatter
+    class Json < Base
+      def call(severity, time, progname, message)
+        super
+        @event.to_json
+      end
+    end
+  end
+end

lib/logstash-logger/formatter/json_lines.rb

@@ -0,0 +1,10 @@
+module LogStashLogger
+  module Formatter
+    class JsonLines < Base
+      def call(severity, time, progname, message)
+        super
+        "#{@event.to_json}\n"
+      end
+    end
+  end
+end

lib/logstash-logger/formatter/logstash_event.rb

@@ -0,0 +1,9 @@
+module LogStashLogger
+  module Formatter
+    class LogStashEvent < Base
+      def call(severity, time, progname, message)
+        super
+      end
+    end
+  end
+end

lib/logstash-logger/logger.rb

@@ -4,13 +4,14 @@
 module LogStashLogger
   def self.new(*args)
     opts = extract_opts(*args)
+    formatter = Formatter.new(opts.delete(:formatter))
     device = Device.new(opts)
 
     ::Logger.new(device).tap do |logger|
       logger.instance_variable_set(:@device, device)
       logger.extend(self)
       logger.extend(TaggedLogging)
-      logger.formatter = Formatter.new
+      logger.formatter = formatter
     end
   end
 

spec/formatter/base_spec.rb

@@ -0,0 +1,72 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter::Base do
+  include_context "formatter"
+
+  describe "#build_event" do
+    let(:event) { formatted_message }
+
+    describe "message type" do
+      context "string" do
+        it "puts the message into the message field" do
+          expect(event['message']).to eq(message)
+        end
+      end
+
+      context "JSON string" do
+        let(:message) do
+          { message: 'test', foo: 'bar' }.to_json
+        end
+
+        it "parses the JSON and merges into the event" do
+          expect(event['message']).to eq('test')
+          expect(event['foo']).to eq('bar')
+        end
+      end
+
+      context "hash" do
+        let(:message) do
+          { 'message' => 'test', 'foo' => 'bar' }
+        end
+
+        it "merges into the event" do
+          expect(event['message']).to eq('test')
+          expect(event['foo']).to eq('bar')
+        end
+      end
+
+      context "LogStash::Event" do
+        let(:message) { LogStash::Event.new("message" => "foo") }
+
+        it "returns a clone of the original event" do
+          expect(event['message']).to eq("foo")
+          expect(event).to_not equal(message)
+        end
+      end
+
+      context "fallback" do
+        let(:message) { [1, 2, 3] }
+
+        it "calls inspect" do
+          expect(event['message']).to eq(message.inspect)
+        end
+      end
+    end
+
+    describe "extra fields on the event" do
+      it "adds severity" do
+        expect(event['severity']).to eq(severity)
+      end
+
+      it "adds host" do
+        expect(event['host']).to eq(hostname)
+      end
+    end
+
+    describe "timestamp" do
+      it "ensures time is in ISO8601 format" do
+        expect(event.timestamp).to eq(time.iso8601(3))
+      end
+    end
+  end
+end

spec/formatter/cee_spec.rb

@@ -0,0 +1,15 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter::Cee do
+  include_context "formatter"
+
+  it "outputs in CEE format" do
+    expect(formatted_message).to match(/\A@cee:/)
+  end
+
+  it "serializes the LogStash::Event data as JSON" do
+    json_data = formatted_message[/\A@cee:\s?(.*)\z/, 1]
+    json_message = JSON.parse(json_data)
+    expect(json_message["message"]).to eq(message)
+  end
+end