Initial working commit

Gemfile

@@ -0,0 +1,6 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+gem "jls-grok"
+gem "rspec"
+gem "rake"

Gemfile.lock

@@ -0,0 +1,26 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cabin (0.4.4)
+      json
+    diff-lcs (1.1.3)
+    jls-grok (0.10.7)
+      cabin (~> 0.4.0)
+    json (1.7.5)
+    rake (0.9.2.2)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.2)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jls-grok
+  rake
+  rspec

README.md

@@ -0,0 +1,49 @@
+[Logstash](http://logstash.net/) is a fantastic tool for processing logs
+and making the information contained in them useful. One of the ways
+this happens is via
+[Grok](http://code.google.com/p/semicomplete/wiki/Grok) patterns. This
+repo is me starting to collect useful patterns. Currently this
+repository contains patterns for:
+
+* [Lograge](https://github.com/roidrage/lograge) - making Rails logs suck less
+
+## Tests
+
+As well as being a set of patterns for people to use in logstash, this
+repo also contains test suites for those patterns and a rspec matcher
+that might be useful to anyone else who wants to write and test grok
+patterns. You can run the test suite with:
+
+    bundle install
+    bundle exec rake
+
+## Contributing
+
+1. Fork this repository
+2. Add a pattern to the patterns directory
+3. Add passing tests to the spec directory
+4. Update the list of patterns in the README
+5. Send a pull request
+
+## License
+
+Copyright (c) 2012 Gareth Rushgrove
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 

Rakefile

@@ -0,0 +1,9 @@
+require 'rake'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = 'spec/*_spec.rb'
+  t.rspec_opts = "--color --format documentation"
+end
+
+task :default => [:spec]

patterns/lograge

@@ -0,0 +1 @@
+LOGRAGE %{WORD:method}%{SPACE}%{DATA}%{SPACE}action=%{WORD:controller}#%{WORD:action}%{SPACE}status=%{INT:status}%{SPACE}duration=%{NUMBER:duration}%{SPACE}view=%{NUMBER:view}%{GREEDYDATA}

patterns/logstash

@@ -0,0 +1,96 @@
+USERNAME [a-zA-Z0-9_-]+
+USER %{USERNAME}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
+QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"]+)*"|(?:'(?:\\.|[^\\']+)*')|(?:`(?:\\.|[^\\`]+)*`)))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+# Networking
+MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IP (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+HOST %{HOSTNAME}
+IPORHOST (?:%{HOSTNAME}|%{IP})
+HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (?:/(?:[\w_%!$@:.,-]+|\\.)*)+
+NUXTTY (?:/dev/pts/%{NONNEGINT})
+BSDTTY (?:/dev/tty[pq][a-z0-9])
+TTY (?:%{BSDTTY}|%{LINUXTTY})
+WINPATH (?:[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=#%_-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~#%&/=:;_-]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR [0-9]+
+# Time: HH:MM:SS
+#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
+# I'm still on the fence about using grok to perform the time match,
+# since it's probably slower.
+# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
+HOUR (?:2[0123]|[01][0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5][0-9]|60)(?:[.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{YEAR}[/-]%{MONTHNUM}[/-]%{MONTHDAY}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
+ISO8601_SECOND (?:%{SECOND}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG (?:[\w._/%-]+)
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{POSINT:facility}.%{POSINT:priority}>
+HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT:ZONE}
+
+# Shortcuts
+QS %{QUOTEDSTRING}
+
+# Log formats
+SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
+COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{URIPATHPARAM:request}(?: HTTP/%{NUMBER:httpversion})?|-)" %{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent}
+
+# Log Levels
+LOGLEVEL ([D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL)/*#UNIXPATH (?<![\w*/

spec/lograge_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe "the lograge grok pattern" do
+
+  before do
+    @grok = Grok.new
+    @grok.add_patterns_from_file("patterns/logstash")
+    @grok.add_patterns_from_file("patterns/lograge")
+    @grok.compile('%{LOGRAGE}')
+  end
+
+  describe "with a standard lograge log line" do
+    before do
+      log_line = "GET /jobs/833552.json format=json action=jobs#show status=200 duration=58.33 view=40.43 db=15.26"
+      @match = @grok.match(log_line)
+    end
+
+    it "should have the correct http method value" do
+      @match.should have_logstash_field("method").with_value("GET")
+    end
+
+    it "should have the correct value for the request duration" do
+      @match.should have_logstash_field("duration").with_value("58.33")
+    end
+
+    it "should have the correct value for the request view time" do
+      @match.should have_logstash_field("view").with_value("40.43")
+    end
+
+    it "should have the correct controller and action" do
+      @match.should have_logstash_field("controller").with_value("jobs")
+      @match.should have_logstash_field("action").with_value("show")
+    end
+  end
+
+  describe "with a post request" do
+    before do
+      log_line = "POST /jobs/833552.json format=json action=jobs#show status=200 duration=58.33 view=40.43 db=15.26"
+      @match = @grok.match(log_line)
+    end
+
+    it "should have the correct http method value" do
+      @match.should have_logstash_field("method").with_value("POST")
+    end
+  end
+
+end

spec/spec_helper.rb

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'grok-pure'
+require 'rspec'
+
+RSpec::Matchers.define :have_logstash_field do |field|
+  chain :with_value do |value|
+    @value= value
+  end
+
+  match do |grok_match|
+    grok_captures = grok_match.captures()
+    grok_field = grok_captures.keys.grep(/.*:#{field}$/)
+    if @value
+      grok_captures[grok_field.first].first.should == @value
+    else
+      grok_field.size.should equal 1
+    end
+  end
+
+  failure_message_for_should do |grok_match|
+    if @value
+      "\"#{field}\" isn't set to \"#{@value}\" in #{grok_match.captures()}"
+    else
+      "\"#{field}\" suffix not present in #{grok_match.captures().keys}"
+    end
+  end
+end