Permalink
Browse files

Updated verbiage of docs, code and specs so that a log_format contain…

…s a log_pattern
  • Loading branch information...
1 parent d9836a6 commit 04bbadcaa1e8da4870389a05e6931a3f8f417ad0 @bfaloona committed Aug 2, 2011
View
@@ -4,7 +4,7 @@ Indy: A Log Archaeology Tool
Synopsis
--------
-Log files are often searched for particular strings but it does not often treat the logs themselves as data structures. Indy attempts to deliver logs with more powerful features by allowing the ability to collect segments of a log from particular time; find a particular event; or monitor/reflect on a log to see if a particular event occurred (or not occurred).
+Log files are often searched for particular strings but are not often treated as data structures. Indy attempts to deliver log content via more powerful features by allowing the ability to collect segments of a log from particular time; find a particular event; or monitor/reflect on a log to see if a particular event occurred (or not occurred).
Installation
------------
@@ -43,42 +43,47 @@ Usage
## Log Pattern
-### Default Log Pattern
+### Default Log Format
-The default search pattern follows this form:
+The default log format follows this form:
YYYY-MM-DD HH:MM:SS SEVERITY APPLICATION_NAME - MESSAGE
-Which uses this regexp:
+Which uses this Regexp:
/^(\d{4}.\d{2}.\d{2}\s+\d{2}.\d{2}.\d{2})\s+(TRACE|DEBUG|INFO|WARN|ERROR|FATAL)\s+(\w+)\s+-\s+(.+)$/
and specifies these fields:
[:time, :severity, :application, :message]
For example:
- Indy.search(source).for(:severity => 'INFO')
- Indy.search(source).for(:application => 'MyApp', :severity => 'DEBUG')
+ Indy.search(log_file).for(:severity => 'INFO')
+ Indy.search(log_file).for(:application => 'MyApp', :severity => 'DEBUG')
-### Custom Log Pattern
+### Custom Log Format
-If the default pattern is obviously not strong enough for you, brew your own.
-To do so, specify a pattern and each of the match with their symbolic name.
+If you have a different log format you can brew your own.
+To do so, specify a Regexp pattern that captures each field you want to reference.
+Include it as the first item of your log format array, followed by a list of symbols that name the captured fields.
+ # If your log format is:
# HH:MM:SS SEVERITY APPLICATION#METHOD - MESSAGE
- custom_pattern = /^(\d{2}:\d{2}:\d{2})\s*(INFO|DEBUG|WARN|ERROR)\s*([^#]+)#([^\s]+)\s*-\s*(.+)$/
-
- Indy.search(source).with(custom_pattern,:time,:severity,:application,:method,:message).for(:severity => 'INFO', :method => 'allocate')
+ # Build an appropriate regexp
+ custom_regexp = /^(\d{2}:\d{2}:\d{2})\s*(INFO|DEBUG|WARN|ERROR)\s*([^#]+)#([^\s]+)\s*-\s*(.+)$/
+ # Combine the pattern and the list of fields
+ custom_log_format = [custom_regexp,:time,:severity,:application,:method,:message]
+ # Use Indy#with to define your format
+ Indy.search(source).with(custom_log_format).for(:severity => 'INFO', :method => 'allocate')
-### Predefined Log Patterns
+### Predefined Log Format
-Several log formats have been predefined for ease of configuration. See indy/patterns.rb
+Several log formats have been predefined for ease of configuration. See indy/formats.rb
- # Indy::COMMON_LOG_PATTERN
- # Indy::COMBINED_LOG_PATTERN
- # Indy::LOG4R_DEFAULT_PATTERN
+ # Indy::COMMON_LOG_FORMAT
+ # Indy::COMBINED_LOG_FORMAT
+ # Indy::LOG4R_DEFAULT_FORMAT
#
# Example (Log4r)
# INFO mylog: This is a message with level INFO
- Indy.new(:source => 'logfile.txt', :pattern => Indy::LOG4R_DEFAULT_PATTERN).for(:application => 'mylog')
+ Indy.new(:source => log_file, :format => Indy::LOG4R_DEFAULT_FORMAT).for(:application => 'mylog')
### Multiline log entries
@@ -109,7 +114,7 @@ Example:
# /^(#{severity_string}) (\w+) - (.*)$/
multiline_regexp = /^(#{severity_string}) (\w+) - (.*?)(?=^#{severity_string}|\z)/
- Indy.new( :multiline => true, :pattern => [multiline_regexp, :severity, :application, :message], :source => MY_LOG)
+ Indy.new( :multiline => true, :format => [multiline_regexp, :severity, :application, :message], :source => MY_LOG)
### Explicit Time Format
@@ -1,4 +1,5 @@
require 'simplecov'
SimpleCov.start if ENV["COVERAGE"]
+SimpleCov.root("#{File.dirname(__FILE__)}/../../../..")
require "#{File.dirname(__FILE__)}/../../../lib/indy"
View
@@ -5,4 +5,3 @@
require 'indy/indy'
require 'indy/result_set'
require 'indy/log_formats'
-require 'indy/patterns'
View
@@ -0,0 +1,3 @@
+class Indy
+
+end
View
@@ -2,6 +2,13 @@
class Indy
+ def self.suppress_warnings(&block)
+ verbose = $VERBOSE
+ $VERBOSE = nil
+ yield block
+ $VERBOSE = verbose
+ end
+
class InvalidSource < Exception; end
VERSION = "0.2.0"
@@ -11,7 +18,7 @@ class InvalidSource < Exception; end
# array with regexp string and capture groups followed by log field
# name symbols. :time field is required to use time scoping
- attr_accessor :pattern
+ attr_accessor :log_format
# format string for explicit date/time format (optional)
attr_accessor :time_format
@@ -24,20 +31,20 @@ class InvalidSource < Exception; end
#
# @example
#
- # Indy.new(:source => LOG_FILENAME)
+ # Indy.new(:source => LOG_FILE)
# Indy.new(:source => LOG_CONTENTS_STRING)
# Indy.new(:source => {:cmd => LOG_COMMAND_STRING})
- # Indy.new(:pattern => [LOG_REGEX_PATTERN,:time,:application,:message],:source => LOG_FILENAME)
- # Indy.new(:time_format => '%m-%d-%Y',:pattern => [LOG_REGEX_PATTERN,:time,:application,:message],:source => LOG_FILENAME)
+ # Indy.new(:log_format => [LOG_REGEX_PATTERN,:time,:application,:message],:source => LOG_FILE)
+ # Indy.new(:time_format => '%m-%d-%Y',:pattern => [LOG_REGEX_PATTERN,:time,:application,:message],:source => LOG_FILE)
#
def initialize(args)
- @source = @pattern = @time_format = @log_regexp = @log_fields = @multiline = nil
+ @source = @log_format = @time_format = @log_regexp = @log_fields = @multiline = nil
while (arg = args.shift) do
send("#{arg.first}=",arg.last)
end
- update_log_pattern( @pattern )
+ update_log_format( @log_format )
end
@@ -72,14 +79,14 @@ class << self
# Indy.search(:cmd => "cat apache.log").for(:severity => "INFO")
#
# @example source as well as other paramters
- # Indy.search(:source => {:cmd => "cat apache.log"}, :pattern => LOG_PATTERN, :time_format => MY_TIME_FORMAT).for(:all)
+ # Indy.search(:source => {:cmd => "cat apache.log"}, :log_format => LOG_FORMAT, :time_format => MY_TIME_FORMAT).for(:all)
#
def search(params=nil)
if params.respond_to?(:keys) && params[:source]
Indy.new(params)
else
- Indy.new(:source => params, :pattern => DEFAULT_LOG_PATTERN)
+ Indy.new(:source => params, :log_format => DEFAULT_LOG_FORMAT)
end
end
@@ -97,19 +104,19 @@ def create_struct( line_hash )
#
- # Specify the log pattern to use as the comparison against each line within
+ # Specify the log format to use as the comparison against each line within
# the log file that has been specified.
#
- # @param [Array] pattern_array an Array with the regular expression as the first element
+ # @param [Array] log_format an Array with the regular expression as the first element
# followed by list of fields (Symbols) in the log entry
# to use for comparison against each log line.
#
# @example Log formatted as - HH:MM:SS Message
#
# Indy.search(LOG_FILE).with(/^(\d{2}.\d{2}.\d{2})\s*(.+)$/,:time,:message)
#
- def with(pattern_array = :default)
- update_log_pattern( pattern_array )
+ def with(log_format = :default)
+ update_log_format( log_format )
self
end
@@ -302,16 +309,16 @@ def within(scope_criteria)
# followed by list of fields (Symbols) in the log entry
# to use for comparison against each log line.
#
- def update_log_pattern( pattern_array )
+ def update_log_format( log_format )
- case pattern_array
+ case log_format
when :default, nil
- @pattern = DEFAULT_LOG_PATTERN
+ @log_format = DEFAULT_LOG_FORMAT
else
- @pattern = pattern_array
+ @log_format = log_format
end
- @log_regexp, *@log_fields = @pattern
+ @log_regexp, *@log_fields = @log_format
@time_field = ( @log_fields.include?(:time) ? :time : nil )
@@ -366,7 +373,7 @@ def _search(&block)
end
- warn "No matching lines found in source: #{source_io.class}" unless line_matched
+# warn "No matching lines found in source: #{source_io.class}" unless line_matched
results.compact
end
@@ -381,7 +388,8 @@ def _search(&block)
def parse_line( line )
if line.kind_of? String
- match_data = /#{@log_regexp}/.match(line)
+ match_data = nil
+ Indy.suppress_warnings { match_data = /#{@log_regexp}/.match(line) }
return nil unless match_data
values = match_data.captures
@@ -494,14 +502,7 @@ def forever_ago
#
def define_struct
fields = (@log_fields + [:_time, :line]).sort_by{|e|e.to_s}
-
- # suppress Struct 'redefining constant' warning
- verbose = $VERBOSE
- $VERBOSE = nil
-
- Struct.new( "Line", *fields )
-
- $VERBOSE = verbose
+ Indy.suppress_warnings { Struct.new( "Line", *fields ) }
end
#
@@ -3,10 +3,9 @@ class Indy
#
# LogFormats defines the building blocks of Indy's built in log patterns.
#
- # See indy/patterns.rb for the following Constants:
- # Indy::COMMON_LOG_PATTERN
- # Indy::COMBINED_LOG_PATTERN
- # Indy::LOG4R_DEFAULT_PATTERN
+ # Indy::COMMON_LOG_FORMAT
+ # Indy::COMBINED_LOG_FORMAT
+ # Indy::LOG4R_DEFAULT_FORMAT
#
module LogFormats
@@ -36,4 +35,27 @@ module LogFormats
LOG4R_DEFAULT_FIELDS = [:level, :application, :message]
LOG4R_DEFAULT_REGEXP = /(?:\s+)?([A-Z]+) (\S+): (.*)$/
end
+
+ #
+ # Indy default log format @pattern
+ # e.g.:
+ # INFO 2000-09-07 MyApp - Entering APPLICATION.
+ #
+ DEFAULT_LOG_FORMAT = [LogFormats::DEFAULT_LOG_REGEXP, LogFormats::DEFAULT_LOG_FIELDS].flatten
+
+ #
+ # Uncustomized Log4r log @pattern
+ #
+ LOG4R_DEFAULT_FORMAT = [LogFormats::LOG4R_DEFAULT_REGEXP, LogFormats::LOG4R_DEFAULT_FIELDS].flatten
+
+ #
+ # NCSA Common Log Format log @pattern
+ #
+ COMMON_LOG_FORMAT = [LogFormats::COMMON_REGEXP, LogFormats::COMMON_FIELDS].flatten
+
+ #
+ # NCSA Combined Log Format log @pattern
+ #
+ COMBINED_LOG_FORMAT = [LogFormats::COMBINED_REGEXP, LogFormats::COMBINED_FIELDS].flatten
+
end
View
@@ -1,25 +0,0 @@
-class Indy
-
- #
- # Indy default log format @pattern
- # e.g.:
- # INFO 2000-09-07 MyApp - Entering APPLICATION.
- #
- DEFAULT_LOG_PATTERN = [LogFormats::DEFAULT_LOG_REGEXP, LogFormats::DEFAULT_LOG_FIELDS].flatten
-
- #
- # Uncustomized Log4r log @pattern
- #
- LOG4R_DEFAULT_PATTERN = [LogFormats::LOG4R_DEFAULT_REGEXP, LogFormats::LOG4R_DEFAULT_FIELDS].flatten
-
- #
- # NCSA Common Log Format log @pattern
- #
- COMMON_LOG_PATTERN = [LogFormats::COMMON_REGEXP, LogFormats::COMMON_FIELDS].flatten
-
- #
- # NCSA Combined Log Format log @pattern
- #
- COMBINED_LOG_PATTERN = [LogFormats::COMBINED_REGEXP, LogFormats::COMBINED_FIELDS].flatten
-
-end
@@ -37,7 +37,7 @@
before(:all) do
@indy = Indy.new(
:source => large_file,
- :pattern => [/^\[([^\|]+)\|([^\]]+)\] (.*)$/,:severity, :time, :message],
+ :log_format => [/^\[([^\|]+)\|([^\]]+)\] (.*)$/,:severity, :time, :message],
:time_format => '%d-%m-%Y %H:%M:%S')
end
View
@@ -1,5 +1,6 @@
require 'simplecov'
SimpleCov.start if ENV["COVERAGE"]
+SimpleCov.root("#{File.dirname(__FILE__)}/../..")
require File.expand_path("#{File.dirname(__FILE__)}/../lib/indy") unless
$:.include? File.expand_path("#{File.dirname(__FILE__)}/../lib/indy")
View
@@ -6,29 +6,29 @@
# http://log4r.rubyforge.org/rdoc/Log4r/rdoc/patternformatter.html
it "should accept a log4r pattern string without error" do
- lambda { Indy.new(:pattern => ["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message]) }.should_not raise_error
+ lambda { Indy.new(:log_format => ["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message]) }.should_not raise_error
end
# http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html
it "should accept a log4j pattern string without error" do
- lambda { Indy.new(:pattern => ["%d [%M] %p %C{1} - %m", :time, :info, :class, :message])}
+ lambda { Indy.new(:log_format => ["%d [%M] %p %C{1} - %m", :time, :info, :class, :message])}
end
it "should not raise error with non-conforming data" do
- @indy = Indy.new(:source => " \nfoobar\n\n baz", :pattern => ['([^\s]+) (\w+)', :time, :message])
+ @indy = Indy.new(:source => " \nfoobar\n\n baz", :log_format => ['([^\s]+) (\w+)', :time, :message])
lambda{ @indy.for(:all) }.should_not raise_error
end
it "should accept time_format parameter" do
- @indy = Indy.new(:time_format => '%d-%m-%Y', :source => "1-13-2000 yes", :pattern => ['^([^\s]+) (\w+)$', :time, :message])
+ @indy = Indy.new(:time_format => '%d-%m-%Y', :source => "1-13-2000 yes", :log_format => ['^([^\s]+) (\w+)$', :time, :message])
lambda{ @indy.for(:all) }.should_not raise_error
@indy.instance_variable_get(:@time_format).should == '%d-%m-%Y'
end
it "should accept an initialization hash passed to #search" do
hash = {:time_format => '%d-%m-%Y',
:source => "1-13-2000 yes",
- :pattern => ['^([^\s]+) (\w+)$', :time, :message]}
+ :log_format => ['^([^\s]+) (\w+)$', :time, :message]}
lambda{ @indy = Indy.search( hash ) }.should_not raise_error
@indy.for(:all).length.should == 1
end
@@ -39,7 +39,7 @@
context 'instance' do
before(:all) do
- @indy = Indy.new(:source => '1/2/2002 string', :pattern => ['([^\s]+) (\w+)', :time, :message])
+ @indy = Indy.new(:source => '1/2/2002 string', :log_format => ['([^\s]+) (\w+)', :time, :message])
end
context "method" do
@@ -189,7 +189,7 @@
"2000-09-07 14:07:42 DEBUG MyApp - Initializing APPLICATION.",
"2000-09-07 14:07:43 INFO MyApp - Exiting APPLICATION with data:\nApplications data\nMore data\n\tlast Application data."].join("\n")
regexp = "^((#{Indy::LogFormats::DEFAULT_DATE_TIME})\\s+(#{Indy::LogFormats::DEFAULT_SEVERITY_PATTERN})\\s+(#{Indy::LogFormats::DEFAULT_APPLICATION})\\s+-\\s+(.*?)(?=#{Indy::LogFormats::DEFAULT_DATE_TIME}|\\z))"
- @indy = Indy.new(:source => log, :pattern => [regexp, :time,:severity,:application,:message], :multiline => true )
+ @indy = Indy.new(:source => log, :log_format => [regexp, :time,:severity,:application,:message], :multiline => true )
end
it "should find the first row" do
@@ -257,7 +257,7 @@ def log
end
it "with() should accept the log4r default pattern const without error" do
- lambda { @indy.with(Indy::LOG4R_DEFAULT_PATTERN) }.should_not raise_error
+ lambda { @indy.with(Indy::LOG4R_DEFAULT_FORMAT) }.should_not raise_error
end
it "with() should accept :default without error" do
Oops, something went wrong.

0 comments on commit 04bbadc

Please sign in to comment.