Skip to content
Browse files

Me being a whitespace nazi.

  • Loading branch information...
1 parent f4c0f33 commit b824c58c4b1f49219a6b135edaf2a8000f15c436 @wvanbergen wvanbergen committed Sep 22, 2009
Showing with 1,019 additions and 1,018 deletions.
  1. +3 −3 LICENSE
  2. +1 −1 README.rdoc
  3. +9 −9 bin/request-log-analyzer
  4. +51 −51 lib/cli/command_line_arguments.rb
  5. +3 −3 lib/cli/database_console.rb
  6. +2 −2 lib/cli/database_console_init.rb
  7. +10 −10 lib/cli/progressbar.rb
  8. +3 −3 lib/cli/tools.rb
  9. +2 −2 lib/request_log_analyzer.rb
  10. +10 −10 lib/request_log_analyzer/aggregator.rb
  11. +9 −9 lib/request_log_analyzer/aggregator/database_inserter.rb
  12. +5 −5 lib/request_log_analyzer/aggregator/echo.rb
  13. +26 −26 lib/request_log_analyzer/aggregator/summarizer.rb
  14. +34 −34 lib/request_log_analyzer/controller.rb
  15. +13 −13 lib/request_log_analyzer/database.rb
  16. +17 −17 lib/request_log_analyzer/database/base.rb
  17. +3 −3 lib/request_log_analyzer/database/connection.rb
  18. +2 −2 lib/request_log_analyzer/database/request.rb
  19. +1 −1 lib/request_log_analyzer/database/source.rb
  20. +14 −14 lib/request_log_analyzer/file_format.rb
  21. +13 −13 lib/request_log_analyzer/file_format/amazon_s3.rb
  22. +12 −12 lib/request_log_analyzer/file_format/apache.rb
  23. +11 −11 lib/request_log_analyzer/file_format/merb.rb
  24. +4 −4 lib/request_log_analyzer/file_format/rack.rb
  25. +18 −18 lib/request_log_analyzer/file_format/rails.rb
  26. +15 −15 lib/request_log_analyzer/file_format/rails_development.rb
  27. +6 −6 lib/request_log_analyzer/filter.rb
  28. +6 −6 lib/request_log_analyzer/filter/anonymize.rb
  29. +9 −9 lib/request_log_analyzer/filter/field.rb
  30. +10 −10 lib/request_log_analyzer/filter/timespan.rb
  31. +14 −14 lib/request_log_analyzer/line_definition.rb
  32. +22 −22 lib/request_log_analyzer/log_processor.rb
  33. +7 −7 lib/request_log_analyzer/mailer.rb
  34. +13 −13 lib/request_log_analyzer/output.rb
  35. +37 −37 lib/request_log_analyzer/output/fixed_width.rb
  36. +20 −20 lib/request_log_analyzer/output/html.rb
  37. +35 −35 lib/request_log_analyzer/request.rb
  38. +7 −7 lib/request_log_analyzer/source.rb
  39. +7 −7 lib/request_log_analyzer/source/database_loader.rb
  40. +42 −42 lib/request_log_analyzer/source/log_parser.rb
  41. +15 −15 lib/request_log_analyzer/tracker.rb
  42. +20 −20 lib/request_log_analyzer/tracker/duration.rb
  43. +10 −10 lib/request_log_analyzer/tracker/frequency.rb
  44. +7 −7 lib/request_log_analyzer/tracker/hourly_spread.rb
  45. +11 −11 lib/request_log_analyzer/tracker/timespan.rb
  46. +21 −21 lib/request_log_analyzer/tracker/traffic.rb
  47. +11 −11 request-log-analyzer.gemspec
  48. +1 −1 spec/integration/command_line_usage_spec.rb
  49. +7 −7 spec/lib/helpers.rb
  50. +3 −3 spec/lib/macros.rb
  51. +11 −11 spec/lib/matchers.rb
  52. +14 −14 spec/lib/mocks.rb
  53. +9 −9 spec/lib/testing_format.rb
  54. +6 −6 spec/spec_helper.rb
  55. +13 −13 spec/unit/aggregator/database_inserter_spec.rb
  56. +4 −4 spec/unit/aggregator/summarizer_spec.rb
  57. +7 −7 spec/unit/controller/controller_spec.rb
  58. +1 −1 spec/unit/controller/log_processor_spec.rb
  59. +19 −19 spec/unit/database/base_class_spec.rb
  60. +3 −3 spec/unit/database/connection_spec.rb
  61. +25 −25 spec/unit/database/database_spec.rb
  62. +5 −5 spec/unit/file_format/amazon_s3_format_spec.rb
  63. +13 −13 spec/unit/file_format/apache_format_spec.rb
  64. +13 −13 spec/unit/file_format/file_format_api_spec.rb
  65. +15 −15 spec/unit/file_format/line_definition_spec.rb
  66. +10 −10 spec/unit/file_format/merb_format_spec.rb
  67. +37 −37 spec/unit/file_format/rails_format_spec.rb
  68. +2 −2 spec/unit/filter/anonymize_filter_spec.rb
  69. +13 −13 spec/unit/filter/field_filter_spec.rb
  70. +1 −1 spec/unit/filter/filter_spec.rb
  71. +15 −15 spec/unit/filter/timespan_filter_spec.rb
  72. +27 −27 spec/unit/source/log_parser_spec.rb
  73. +29 −29 spec/unit/source/request_spec.rb
  74. +13 −13 spec/unit/tracker/duration_tracker_spec.rb
  75. +13 −13 spec/unit/tracker/frequency_tracker_spec.rb
  76. +19 −19 spec/unit/tracker/hourly_spread_spec.rb
  77. +13 −13 spec/unit/tracker/timespan_tracker_spec.rb
  78. +13 −13 spec/unit/tracker/tracker_api_spec.rb
  79. +22 −21 spec/unit/tracker/traffic_tracker_spec.rb
  80. +2 −2 tasks/request_log_analyzer.rake
View
6 LICENSE
@@ -1,16 +1,16 @@
Copyright (c) 2008 Willem van Bergen / Bart ten Brinke
-
+
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
-
+
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
-
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
View
2 README.rdoc
@@ -21,7 +21,7 @@ Alternatively, use the gem from the GitHub gem server:
$ sudo gem install wvanbergen-request-log-analyzer --source http://gems.github.com
-To get the best results out of request-log-analyzer, make sure to
+To get the best results out of request-log-analyzer, make sure to
set up logging correctly: http://wiki.github.com/wvanbergen/request-log-analyzer/configure-logging
for your application.
View
18 bin/request-log-analyzer
@@ -24,7 +24,7 @@ begin
strip.option(:format, :alias => :f, :default => 'rails')
strip.option(:output, :alias => :o)
strip.switch(:discard_teaser_lines, :t)
- strip.switch(:keep_junk_lines, :j)
+ strip.switch(:keep_junk_lines, :j)
end
command_line.option(:format, :alias => :f, :default => 'rails')
@@ -57,7 +57,7 @@ begin
end
rescue CommandLine::Error => e
- puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"
+ puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"
puts "Website: http://railsdoctors.com"
puts
puts "ARGUMENT ERROR: " + e.message if e.message
@@ -67,19 +67,19 @@ rescue CommandLine::Error => e
puts "Input options:"
puts " --format <format>, -f: Uses the specified log file format. Defaults to rails."
puts " --after <date> Only consider requests from <date> or later."
- puts " --before <date> Only consider requests before <date>."
- puts " --select <field> <value> Only consider requests where <field> matches <value>."
- puts " --reject <field> <value> Only consider requests where <field> does not match <value>."
+ puts " --before <date> Only consider requests before <date>."
+ puts " --select <field> <value> Only consider requests where <field> matches <value>."
+ puts " --reject <field> <value> Only consider requests where <field> does not match <value>."
puts
puts "Output options:"
puts " --boring, -b Output reports without ASCII colors."
puts " --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
puts " --debug Print debug information while parsing."
puts " --file <filename> Output to file."
puts " --mail <emailaddress> Send report to an email address."
- puts " --output <format> Output format. Supports 'HTML' and 'FixedWidth' (default)"
+ puts " --output <format> Output format. Supports 'HTML' and 'FixedWidth' (default)"
puts " --dump <filename> Dump the YAML formatted results in the given file"
- puts
+ puts
puts "Examples:"
puts " request-log-analyzer development.log"
puts " request-log-analyzer -b mongrel.0.log mongrel.1.log mongrel.2.log "
@@ -89,7 +89,7 @@ rescue CommandLine::Error => e
puts "run the following command in your application's root directory:"
puts
puts " request-log-analyzer install rails"
- exit(0)
+ exit(0)
end
case arguments.command
@@ -101,7 +101,7 @@ when :console
when :strip
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'
RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
-else
+else
puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"
puts "Website: http://railsdoctors.com"
puts
View
102 lib/cli/command_line_arguments.rb
@@ -71,7 +71,7 @@ def has_alias?
def required?
@required
end
-
+
# Check if flag is optional
def optional?
!@required
@@ -85,22 +85,22 @@ def has_default?
!@default_value.nil?
end
end
-
+
class Arguments
class Definition
-
+
ENDLESS_PARAMETERS = 99999
-
+
attr_reader :commands, :options, :parameters
-
+
def initialize(parent)
@parent = parent
@options = {}
@commands = {}
@parameters = nil
end
-
+
def [](option_name)
option_symbol = CommandLine::Option.rewrite(option_name)
if the_option = @options.detect { |(name, odef)| odef =~ option_symbol }
@@ -109,22 +109,22 @@ def [](option_name)
raise CommandLine::UnknownOption, option_name
end
end
-
+
def minimum_parameters=(count_specifier)
@parameters = count_specifier..ENDLESS_PARAMETERS
end
-
+
def parameters=(count_specifier)
@parameters = count_specifier
end
-
- alias :files= :parameters=
-
+
+ alias :files= :parameters=
+
def option(name, options = {})
clo = CommandLine::Option.new(name, options)
@options[clo.name] = clo
end
-
+
def switch(name, switch_alias = nil)
option(name, :alias => switch_alias, :parameters => 0)
end
@@ -134,67 +134,67 @@ def command(name, &block)
yield(command_definition) if block_given?
@commands[CommandLine::Option.rewrite(name)] = command_definition
end
-
+
def has_command?(command)
@commands[CommandLine::Option.rewrite(command)]
end
end
-
+
OPTION_REGEXP = /^\-\-([A-Za-z0-9-]+)$/;
ALIASES_REGEXP = /^\-([A-Aa-z0-9]+)$/
-
+
attr_reader :definition
attr_reader :tokens
attr_reader :command, :options, :parameters
-
+
def self.parse(tokens = $*, &block)
cla = Arguments.new
cla.define(&block)
- return cla.parse!(tokens)
+ return cla.parse!(tokens)
end
-
+
def initialize
@tokens = []
- @definition = Definition.new(self)
- @current_definition = @definition
+ @definition = Definition.new(self)
+ @current_definition = @definition
end
-
+
def define(&block)
yield(@definition)
end
-
+
def [](option)
if the_option = @options.detect { |(key, value)| key =~ option }
the_option[1]
else
@current_definition[option].default_value
end
end
-
+
def next_token
@current_token = @tokens.shift
return @current_token
end
-
+
def next_parameter
parameter_candidate = @tokens.first
parameter = (parameter_candidate.nil? || OPTION_REGEXP =~ parameter_candidate || ALIASES_REGEXP =~ parameter_candidate) ? nil : @tokens.shift
return parameter
end
-
+
def parse!(tokens)
- @current_definition = @definition
+ @current_definition = @definition
@first_token = true
@tokens = tokens.clone
-
+
@options = {}
@parameters = []
@command = nil
-
+
prepare_result!
-
+
while next_token
-
+
if @first_token && command_definition = @definition.has_command?(@current_token)
@current_definition = command_definition
@command = CommandLine::Option.rewrite(@current_token)
@@ -204,34 +204,34 @@ def parse!(tokens)
when OPTION_REGEXP; handle_option($1)
else; handle_other_parameter(@current_token)
end
- @first_token = false
+ @first_token = false
end
-
+
end
-
+
validate_arguments!
-
- return self
+
+ return self
end
-
+
protected
-
+
def prepare_result!
multiple_options = Hash[*@current_definition.options.select { |name, o| o.multiple? }.flatten]
multiple_options.each { |name, definition| @options[definition] = [] }
end
-
+
def validate_arguments!
if @current_definition.parameters && !(@current_definition.parameters === @parameters.length)
raise CommandLine::ParametersOutOfRange.new(@current_definition.parameters, @parameters.length)
end
-
+
required_options = Hash[*@current_definition.options.select { |name, o| o.required? }.flatten]
required_options.each do |name, definition|
- raise CommandLine::RequiredOptionMissing, definition unless self[name]
+ raise CommandLine::RequiredOptionMissing, definition unless self[name]
end
end
-
+
def handle_alias_expansion(aliases)
aliases.reverse.scan(/./) do |alias_char|
if option_definition = @current_definition[alias_char]
@@ -241,24 +241,24 @@ def handle_alias_expansion(aliases)
end
end
end
-
+
def handle_other_parameter(parameter)
@parameters << parameter
end
-
+
def handle_option(option_name)
option_definition = @current_definition[option_name]
raise CommandLine::UnknownOption, option_name if option_definition.nil?
-
+
if option_definition.multiple?
- @options[option_definition] << option_definition.parse(self)
+ @options[option_definition] << option_definition.parse(self)
else
@options[option_definition] = option_definition.parse(self)
end
end
-
+
end
-
+
# Commandline parsing errors and exceptions
class Error < Exception
end
@@ -267,7 +267,7 @@ class Error < Exception
class RequiredOptionMissing < CommandLine::Error
def initialize(option)
super("You have to provide the #{option.name} option!")
- end
+ end
end
# Missing a required file
@@ -282,20 +282,20 @@ def initialize(expected, actual)
else
super("The command expected #{expected} parameters, but found #{actual}!")
end
- end
+ end
end
# Missing a required flag argument
class ParameterExpected < CommandLine::Error
def initialize(option)
super("The option #{option.inspect} expects a parameter!")
- end
+ end
end
# Encountered an unkown flag
class UnknownOption < CommandLine::Error
def initialize(option_identifier)
super("#{option_identifier.inspect} not recognized as a valid option!")
end
- end
+ end
end
View
6 lib/cli/database_console.rb
@@ -1,12 +1,12 @@
class DatabaseConsole
-
+
IRB = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
-
+
def initialize(arguments)
@arguments = arguments
end
-
+
def run!
libraries = ['irb/completion', 'rubygems', './lib/request_log_analyzer', './lib/cli/database_console_init']
libaries_string = libraries.map { |l| "-r #{l}" }.join(' ')
View
4 lib/cli/database_console_init.rb
@@ -23,7 +23,7 @@ class Request
def inspect
request_inspect = "Request[id: #{id}]"
request_inspect << " <#{lines.first.source.filename}>" if lines.first.source
-
+
inspected_lines = lines.map do |line|
inspect_line = " - #{line.line_type} (line #{line.lineno})"
if (inspect_attributes = line.attributes.reject { |(k, v)| [:id, :source_id, :request_id, :lineno].include?(k.to_sym) }).any?
@@ -32,7 +32,7 @@ def inspect
end
inspect_line
end
-
+
request_inspect << "\n" << inspected_lines.join("\n") << "\n\n"
end
end
View
20 lib/cli/progressbar.rb
@@ -37,8 +37,8 @@ def initialize (title, total, out = STDERR)
private
def fmt_bar
bar_width = do_percentage * @terminal_width / 100
- sprintf("[%s%s]",
- @bar_mark * bar_width,
+ sprintf("[%s%s]",
+ @bar_mark * bar_width,
" " * (@terminal_width - bar_width))
end
@@ -51,9 +51,9 @@ def fmt_stat
end
def fmt_stat_for_file_transfer
- if @finished_p then
+ if @finished_p then
sprintf("%s %s %s", bytes, transfer_rate, elapsed)
- else
+ else
sprintf("%s %s %s", bytes, transfer_rate, eta)
end
end
@@ -106,7 +106,7 @@ def elapsed
elapsed = Time.now - @start_time
sprintf("Time: %s", format_time(elapsed))
end
-
+
def eol
if @finished_p then "\n" else "\r" end
end
@@ -120,14 +120,14 @@ def do_percentage
end
def show
- arguments = @format_arguments.map {|method|
+ arguments = @format_arguments.map {|method|
method = sprintf("fmt_%s", method)
send(method)
}
line = sprintf(@format, *arguments)
width = terminal_width(80)
- if line.length == width - 1
+ if line.length == width - 1
@out.print(line + eol)
@out.flush
elsif line.length >= width
@@ -150,7 +150,7 @@ def show_if_needed
end
# Use "!=" instead of ">" to support negative changes
- if cur_percentage != prev_percentage ||
+ if cur_percentage != prev_percentage ||
Time.now - @previous_time >= 1 || @finished_p
show
end
@@ -198,9 +198,9 @@ def inc (step = 1)
end
def set (count)
- count = 0 if count < 0
+ count = 0 if count < 0
count = @total if count > @total
-
+
@current = count
show_if_needed
@previous = @current
View
6 lib/cli/tools.rb
@@ -4,15 +4,15 @@
def terminal_width(default_width = 81)
tiocgwinsz = 0x5413
data = [0, 0, 0, 0].pack("SSSS")
- if @out.ioctl(tiocgwinsz, data) >= 0
+ if @out.ioctl(tiocgwinsz, data) >= 0
rows, cols, xpixels, ypixels = data.unpack("SSSS")
raise unless cols > 0
cols
else
raise
end
-rescue
- begin
+rescue
+ begin
IO.popen('stty -a 2>&1') do |pipe|
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
raise unless column_line
View
4 lib/request_log_analyzer.rb
@@ -8,7 +8,7 @@
# - This module itselfs contains some functions to help with class and source file loading.
# - The actual application resides in the RequestLogAnalyzer::Controller class.
module RequestLogAnalyzer
-
+
# The current version of request-log-analyzer.
# This will be diplayed in output reports etc.
VERSION = "1.3.7"
@@ -35,7 +35,7 @@ def self.to_underscore(str)
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
end
- # Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
+ # Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
# (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.
# <tt>str</tt>:: The string to convert in the following format: <tt>module_name/class_name</tt>
def self.to_camelcase(str)
View
20 lib/request_log_analyzer/aggregator.rb
@@ -1,13 +1,13 @@
module RequestLogAnalyzer::Aggregator
-
+
def self.const_missing(const)
RequestLogAnalyzer::load_default_class_file(self, const)
end
-
+
# The base class of an aggregator. This class provides the interface to which
# every aggregator should comply (by simply subclassing this class).
class Base
-
+
attr_reader :options, :source
# Intializes a new RequestLogAnalyzer::Aggregator::Base instance
@@ -17,30 +17,30 @@ def initialize(source, options = {})
@options = options
end
- # The prepare function is called just before parsing starts. This function
+ # The prepare function is called just before parsing starts. This function
# can be used to initialie variables, etc.
def prepare
end
-
+
# The aggregate function is called for every request.
# Implement the aggregating functionality in this method
def aggregate(request)
end
-
+
# The finalize function is called after all sources are parsed and no more
# requests will be passed to the aggregator
def finalize
end
-
+
# The warning method is called if the parser eits a warning.
def warning(type, message, lineno)
- end
-
+ end
+
# The report function is called at the end. Implement any result reporting
# in this function.
def report(output)
end
-
+
# The source_change function gets called when handling a source is started or finished.
def source_change(change, filename)
end
View
18 lib/request_log_analyzer/aggregator/database_inserter.rb
@@ -4,7 +4,7 @@ module RequestLogAnalyzer::Aggregator
# The database aggregator will create an SQLite3 database with all parsed request information.
#
# The prepare method will create a database schema according to the file format definitions.
- # It will also create ActiveRecord::Base subclasses to interact with the created tables.
+ # It will also create ActiveRecord::Base subclasses to interact with the created tables.
# Then, the aggregate method will be called for every parsed request. The information of
# these requests is inserted into the tables using the ActiveRecord classes.
#
@@ -22,11 +22,11 @@ def prepare
@sources = {}
@database = RequestLogAnalyzer::Database.new(options[:database])
@database.file_format = source.file_format
-
+
database.drop_database_schema! if options[:reset_database]
database.create_database_schema!
end
-
+
# Aggregates a request into the database
# This will create a record in the requests table and create a record for every line that has been parsed,
# in which the captured values will be stored.
@@ -42,19 +42,19 @@ def aggregate(request)
rescue SQLite3::SQLException => e
raise Interrupt, e.message
end
-
+
# Finalizes the aggregator by closing the connection to the database
def finalize
@request_count = RequestLogAnalyzer::Database::Request.count
database.disconnect
database.remove_orm_classes!
end
-
+
# Records w warining in the warnings table.
def warning(type, message, lineno)
RequestLogAnalyzer::Database::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
end
-
+
# Records source changes in the sources table
def source_change(change, filename)
if File.exist?(filename)
@@ -66,18 +66,18 @@ def source_change(change, filename)
end
end
end
-
+
# Prints a short report of what has been inserted into the database
def report(output)
output.title('Request database created')
-
+
output << "A database file has been created with all parsed request information.\n"
output << "#{@request_count} requests have been added to the database.\n"
output << "\n"
output << "To open a Ruby console to inspect the database, run the following command.\n"
output << output.colorize(" $ request-log-analyzer console -d #{options[:database]}\n", :bold)
output << "\n"
end
-
+
end
end
View
10 lib/request_log_analyzer/aggregator/echo.rb
@@ -1,20 +1,20 @@
module RequestLogAnalyzer::Aggregator
- # Echo Aggregator. Writes everything passed to it
+ # Echo Aggregator. Writes everything passed to it
class Echo < Base
-
+
def prepare
@warnings = ""
end
-
+
def aggregate(request)
puts "\nRequest: " + request.inspect
end
-
+
def warning(type, message, lineno)
@warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}\n"
end
-
+
def report(output)
output.title("Warnings during parsing")
output.puts @warnings
View
52 lib/request_log_analyzer/aggregator/summarizer.rb
@@ -1,9 +1,9 @@
module RequestLogAnalyzer::Aggregator
class Summarizer < Base
-
+
class Definer
-
+
attr_reader :trackers
# Initialize tracker array
@@ -16,17 +16,17 @@ def initialize
def initialize_copy(other)
@trackers = other.trackers.dup
end
-
+
# Drop all trackers
def reset!
@trackers = []
end
-
+
# Include missing trackers through method missing.
def method_missing(tracker_method, *args)
track(tracker_method, *args)
end
-
+
# Track the frequency of a specific category
# <tt>category_field</tt> Field to track
# <tt>options</tt> options are passed to new frequency tracker
@@ -37,18 +37,18 @@ def frequency(category_field, options = {})
track(:frequency, category_field.merge(options))
end
end
-
+
# Track the duration of a specific category
# <tt>duration_field</tt> Field to track
# <tt>options</tt> options are passed to new frequency tracker
def duration(duration_field, options = {})
if duration_field.kind_of?(Symbol)
track(:duration, options.merge(:duration => duration_field))
elsif duration_field.kind_of?(Hash)
- track(:duration, duration_field.merge(options))
+ track(:duration, duration_field.merge(options))
end
- end
-
+ end
+
# Helper function to initialize a tracker and add it to the tracker array.
# <tt>tracker_class</tt> The class to include
# <tt>optiont</tt> The options to pass to the trackers.
@@ -57,10 +57,10 @@ def track(tracker_klass, options = {})
@trackers << tracker_klass.new(options)
end
end
-
+
attr_reader :trackers
attr_reader :warnings_encountered
-
+
# Initialize summarizer.
# Generate trackers from speciefied source.file_format.report_trackers and set them up
def initialize(source, options = {})
@@ -69,36 +69,36 @@ def initialize(source, options = {})
@trackers = source.file_format.report_trackers
setup
end
-
+
def setup
end
-
+
# Call prepare on all trackers.
def prepare
raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
@trackers.each { |tracker| tracker.prepare }
end
-
+
# Pass all requests to trackers and let them update if necessary.
# <tt>request</tt> The request to pass.
def aggregate(request)
@trackers.each do |tracker|
tracker.update(request) if tracker.should_update?(request)
end
end
-
+
# Call finalize on all trackers. Saves a YAML dump if this is set in the options.
def finalize
@trackers.each { |tracker| tracker.finalize }
save_results_dump(options[:dump]) if options[:dump]
end
-
+
# Saves the results of all the trackers in YAML format to a file.
# <tt>filename</tt> The file to store the YAML dump in.
def save_results_dump(filename)
File.open(filename, 'w') { |file| file.write(to_yaml) }
end
-
+
# Exports all the tracker results to YAML. It will call the to_yaml_object method
# for every tracker and combines these into a single YAML export.
def to_yaml
@@ -108,7 +108,7 @@ def to_yaml
end
YAML::dump(trackers_export)
end
-
+
# Call report on all trackers.
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
def report(output)
@@ -121,13 +121,13 @@ def report(output)
end
report_footer(output)
end
-
+
# Generate report header.
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
def report_header(output)
output.title("Request summary")
-
- output.with_style(:cell_separator => false) do
+
+ output.with_style(:cell_separator => false) do
output.table({:width => 20}, {:font => :bold}) do |rows|
rows << ['Parsed lines:', source.parsed_lines]
rows << ['Skipped lines:', source.skipped_lines]
@@ -138,31 +138,31 @@ def report_header(output)
end
output << "\n"
end
-
+
# Generate report footer.
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
def report_footer(output)
if has_log_ordering_warnings?
output.title("Parse warnings")
-
+
output.puts "Parseable lines were ancountered without a header line before it. It"
output.puts "could be that logging is not setup correctly for your application."
output.puts "Visit this website for logging configuration tips:"
output.puts output.link("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging")
output.puts
end
end
-
+
# Returns true if there were any warnings generated by the trackers
def has_warnings?
@warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
end
-
+
# Returns true if there were any log ordering warnings
def has_log_ordering_warnings?
@warnings_encountered[:no_current_request] && @warnings_encountered[:no_current_request] > 0
end
-
+
# Store an encountered warning
# <tt>type</tt> Type of warning
# <tt>message</tt> Warning message
View
68 lib/request_log_analyzer/controller.rb
@@ -1,5 +1,5 @@
module RequestLogAnalyzer
-
+
# The RequestLogAnalyzer::Controller class creates a LogParser instance for the
# requested file format, and connect it with sources and aggregators.
#
@@ -32,23 +32,23 @@ def self.build(arguments)
options[:dump] = arguments[:dump]
options[:parse_strategy] = arguments[:parse_strategy]
options[:no_progress] = arguments[:no_progress]
-
+
output_class = RequestLogAnalyzer::Output::const_get(arguments[:output])
if arguments[:file]
output_file = File.new(arguments[:file], "w+")
options[:output] = output_class.new(output_file, :width => 80, :color => false, :characters => :ascii)
elsif arguments[:mail]
output_mail = RequestLogAnalyzer::Mailer.new(arguments[:mail])
- options[:output] = output_class.new(output_mail, :width => 80, :color => false, :characters => :ascii)
+ options[:output] = output_class.new(output_mail, :width => 80, :color => false, :characters => :ascii)
else
- options[:output] = output_class.new(STDOUT, :width => arguments[:report_width].to_i,
+ options[:output] = output_class.new(STDOUT, :width => arguments[:report_width].to_i,
:color => !arguments[:boring], :characters => (arguments[:boring] ? :ascii : :utf))
end
-
+
# Create the controller with the correct file format
file_format = if arguments[:apache_format]
RequestLogAnalyzer::FileFormat.load(:apache, arguments[:apache_format])
- else
+ else
RequestLogAnalyzer::FileFormat.load(arguments[:format])
end
@@ -66,38 +66,38 @@ def self.build(arguments)
else
options.store(:source_files, arguments.parameters)
end
-
+
controller = Controller.new(RequestLogAnalyzer::Source::LogParser.new(file_format, options), options)
#controller = Controller.new(RequestLogAnalyzer::Source::DatabaseLoader.new(file_format, options), options)
-
+
# register filters
if arguments[:after] || arguments[:before]
filter_options = {}
- filter_options[:after] = DateTime.parse(arguments[:after])
+ filter_options[:after] = DateTime.parse(arguments[:after])
filter_options[:before] = DateTime.parse(arguments[:before]) if arguments[:before]
controller.add_filter(:timespan, filter_options)
end
-
+
arguments[:reject].each do |(field, value)|
controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
end
-
+
arguments[:select].each do |(field, value)|
controller.add_filter(:field, :mode => :select, :field => field, :value => value)
end
# register aggregators
arguments[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
- # register the database
+ # register the database
controller.add_aggregator(:summarizer) if arguments[:aggregator].empty?
controller.add_aggregator(:database_inserter) if arguments[:database] && !arguments[:aggregator].include?('database')
# register the echo aggregator in debug mode
controller.add_aggregator(:echo) if arguments[:debug]
-
+
file_format.setup_environment(controller)
-
+
return controller
end
@@ -117,16 +117,16 @@ def initialize(source, options = {})
@aggregators = []
@filters = []
@output = options[:output]
-
+
# Register the request format for this session after checking its validity
raise "Invalid file format!" unless @source.file_format.valid?
-
+
# Install event handlers for wrnings, progress updates and source changes
@source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } }
@source.progress = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
@source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
end
-
+
# Progress function.
# Expects :started with file, :progress with current line and :finished or :interrupted when done.
# <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
@@ -147,46 +147,46 @@ def handle_progress(message, value = nil)
@progress_bar.set(value)
end
end
-
+
# Source change handler
def handle_source_change(change, filename)
@aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) }
end
-
- # Adds an aggregator to the controller. The aggregator will be called for every request
+
+ # Adds an aggregator to the controller. The aggregator will be called for every request
# that is parsed from the provided sources (see add_source)
- def add_aggregator(agg)
+ def add_aggregator(agg)
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer::to_camelcase(agg)) if agg.kind_of?(Symbol)
@aggregators << agg.new(@source, @options)
end
-
+
alias :>> :add_aggregator
-
+
# Adds a request filter to the controller.
def add_filter(filter, filter_options = {})
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
@filters << filter.new(source.file_format, @options.merge(filter_options))
end
-
+
# Push a request through the entire filterchain (@filters).
# <tt>request</tt> The request to filter.
# Returns the filtered request or nil.
def filter_request(request)
- @filters.each do |filter|
+ @filters.each do |filter|
request = filter.filter(request)
return nil if request.nil?
end
return request
end
-
+
# Push a request to all the aggregators (@aggregators).
- # <tt>request</tt> The request to push to the aggregators.
+ # <tt>request</tt> The request to push to the aggregators.
def aggregate_request(request)
return false unless request
@aggregators.each { |agg| agg.aggregate(request) }
return true
end
-
+
# Runs RequestLogAnalyzer
# 1. Call prepare on every aggregator
# 2. Generate requests from source object
@@ -196,10 +196,10 @@ def aggregate_request(request)
# 5. Call report on every aggregator
# 6. Finalize Source
def run!
-
+
@aggregators.each { |agg| agg.prepare }
install_signal_handlers
-
+
@source.each_request do |request|
break if @interrupted
aggregate_request(filter_request(request))
@@ -210,9 +210,9 @@ def run!
@output.header
@aggregators.each { |agg| agg.report(@output) }
@output.footer
-
+
@source.finalize
-
+
if @output.io.kind_of?(File)
puts
puts "Report written to: " + File.expand_path(@output.io.path)
@@ -224,14 +224,14 @@ def run!
@output.io.mail
end
end
-
+
def install_signal_handlers
Signal.trap("INT") do
handle_progress(:interrupted)
puts "Caught interrupt! Stopping parsing..."
@interrupted = true
end
end
-
+
end
end
View
26 lib/request_log_analyzer/database.rb
@@ -5,13 +5,13 @@ class RequestLogAnalyzer::Database
def self.const_missing(const) # :nodoc:
RequestLogAnalyzer::load_default_class_file(self, const)
- end
+ end
include RequestLogAnalyzer::Database::Connection
attr_accessor :file_format
attr_reader :line_classes
-
+
def initialize(connection_identifier = nil)
@line_classes = []
RequestLogAnalyzer::Database::Base.database = self
@@ -23,11 +23,11 @@ def get_class(line_type)
line_type = line_type.name if line_type.respond_to?(:name)
Object.const_get("#{line_type}_line".camelize)
end
-
+
def default_classes
[RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
end
-
+
# Loads the ORM classes by inspecting the tables in the current database
def load_database_schema!
connection.tables.map do |table|
@@ -39,16 +39,16 @@ def load_database_schema!
end
end
end
-
+
# Returns an array of all the ActiveRecord-bases ORM classes for this database
def orm_classes
default_classes + line_classes
end
-
+
# Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
# a table name or a LineDefinition instance.
def load_activerecord_class(linedefinition_or_table)
-
+
case linedefinition_or_table
when String, Symbol
klass_name = linedefinition_or_table.to_s.singularize.camelize
@@ -57,38 +57,38 @@ def load_activerecord_class(linedefinition_or_table)
klass_name = "#{linedefinition_or_table.name}_line".camelize
klass = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
end
-
+
Object.const_set(klass_name, klass)
klass = Object.const_get(klass_name)
@line_classes << klass
return klass
- end
+ end
def fileformat_classes
raise "No file_format provided!" unless file_format
line_classes = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
return default_classes + line_classes
end
- # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
+ # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
# file format definition. These ORM classes will later be used to create records in the database.
def create_database_schema!
fileformat_classes.each { |klass| klass.create_table! }
end
-
+
# Drops the table of all the ORM classes, and unregisters the classes
def drop_database_schema!
file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
remove_orm_classes!
end
-
+
# Registers the default ORM classes in the default namespace
def register_default_orm_classes!
Object.const_set('Request', RequestLogAnalyzer::Database::Request)
Object.const_set('Source', RequestLogAnalyzer::Database::Source)
Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
end
-
+
# Unregisters every ORM class constant
def remove_orm_classes!
orm_classes.each do |klass|
View
34 lib/request_log_analyzer/database/base.rb
@@ -1,5 +1,5 @@
class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
-
+
self.abstract_class = true
def <=>(other)
@@ -13,20 +13,20 @@ def <=>(other)
def line_type
self.class.name.underscore.gsub(/_line$/, '').to_sym
end
-
+
class_inheritable_accessor :line_definition
cattr_accessor :database
def self.subclass_from_line_definition(definition)
klass = Class.new(RequestLogAnalyzer::Database::Base)
klass.set_table_name("#{definition.name}_lines")
-
+
klass.line_definition = definition
-
+
# Set relations with requests and sources table
klass.belongs_to :request
klass.belongs_to :source
-
+
# Serialize complex fields into the database
definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
klass.send(:serialize, capture[:name], Hash)
@@ -48,32 +48,32 @@ def self.subclass_from_table(table)
klass.belongs_to :request
RequestLogAnalyzer::Database::Request.has_many table.to_sym
end
-
+
if klass.column_names.include?('source_id')
klass.belongs_to :source
RequestLogAnalyzer::Database::Source.has_many table.to_sym
end
-
+
return klass
end
-
+
def self.drop_table!
database.connection.remove_index(self.table_name, [:source_id]) rescue nil
database.connection.remove_index(self.table_name, [:request_id]) rescue nil
database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
end
-
+
def self.create_table!
raise "No line_definition available to base table schema on!" unless self.line_definition
-
+
unless table_exists?
database.connection.create_table(table_name.to_sym) do |t|
-
+
# Default fields
t.column :request_id, :integer
t.column :source_id, :integer
t.column :lineno, :integer
-
+
line_definition.captures.each do |capture|
# Add a field for every capture
t.column(capture[:name], column_type(capture[:type]))
@@ -83,13 +83,13 @@ def self.create_table!
end
end
end
-
+
# Add indices to table for more speedy querying
database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
database.connection.add_index(self.table_name.to_sym, [:source_id]) # rescue
end
-
-
+
+
# Function to determine the column type for a field
# TODO: make more robust / include in file-format definition
def self.column_type(type_indicator)
@@ -110,6 +110,6 @@ def self.column_type(type_indicator)
when :date; :date
else :string
end
- end
-
+ end
+
end
View
6 lib/request_log_analyzer/database/connection.rb
@@ -26,13 +26,13 @@ def connect(connection_identifier)
raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
end
end
-
+
def disconnect
RequestLogAnalyzer::Database::Base.remove_connection
end
-
+
def connection
RequestLogAnalyzer::Database::Base.connection
end
-
+
end
View
4 lib/request_log_analyzer/database/request.rb
@@ -1,5 +1,5 @@
class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
-
+
# Returns an array of all the Line objects of this request in the correct order.
def lines
@lines ||= begin
@@ -8,7 +8,7 @@ def lines
lines.sort
end
end
-
+
# Creates the table to store requests in.
def self.create_table!
unless database.connection.table_exists?(:requests)
View
2 lib/request_log_analyzer/database/source.rb
@@ -1,5 +1,5 @@
class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
-
+
def self.create_table!
unless database.connection.table_exists?(:sources)
database.connection.create_table(:sources) do |t|
View
28 lib/request_log_analyzer/file_format.rb
@@ -1,9 +1,9 @@
module RequestLogAnalyzer::FileFormat
-
+
def self.const_missing(const)
RequestLogAnalyzer::load_default_class_file(self, const)
- end
-
+ end
+
# Loads a FileFormat::Base subclass instance.
# You can provide:
# * A FileFormat instance (which will return itself)
@@ -34,46 +34,46 @@ def self.load(file_format, *args)
else
# load a provided file format
- klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))
+ klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))
end
-
+
# check the returned klass to see if it can be used
raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
-
+
@current_file_format = klass.create(*args) # return an instance of the class
end
- # Base class for all log file format definitions. This class provides functions for subclasses to
+ # Base class for all log file format definitions. This class provides functions for subclasses to
# define their LineDefinitions and to define a summary report.
#
# A subclass of this class is instantiated when request-log-analyzer is started and this instance
# is shared with all components of the application so they can act on the specifics of the format
class Base
-
+
attr_reader :line_definitions, :report_trackers
-
+
####################################################################################
# CLASS METHODS for format definition
####################################################################################
-
+
# Registers the line definer instance for a subclass.
def self.inherited(subclass)
if subclass.superclass == RequestLogAnalyzer::FileFormat::Base
# Create aline and report definer for this class
- subclass.class_eval do
+ subclass.class_eval do
instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
instance_variable_set(:@report_definer, RequestLogAnalyzer::Aggregator::Summarizer::Definer.new)
class << self; attr_accessor :line_definer, :report_definer; end
- end
+ end
# Create a custom Request class for this file format
subclass.const_set('Request', Class.new(RequestLogAnalyzer::Request)) unless subclass.const_defined?('Request')
else
# Copy the line and report definer from the parent class.
- subclass.class_eval do
+ subclass.class_eval do
instance_variable_set(:@line_definer, superclass.line_definer.clone)
instance_variable_set(:@report_definer, superclass.report_definer.clone)
class << self; attr_accessor :line_definer, :report_definer; end
@@ -153,7 +153,7 @@ def parse_line(line, &warning_handler)
match = definition.matches(line, &warning_handler)
return match if match
end
-
+
return nil
end
end
View
26 lib/request_log_analyzer/file_format/amazon_s3.rb
@@ -1,20 +1,20 @@
module RequestLogAnalyzer::FileFormat
- # FileFormat for Amazon S3 access logs.
+ # FileFormat for Amazon S3 access logs.
#
# Access logs are disabled by default on Amazon S3. To enable logging, see
# http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
class AmazonS3 < Base
-
+
line_definition :access do |line|
line.header = true
line.footer = true
line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
- line.captures << { :name => :bucket_owner, :type => :string } <<
- { :name => :bucket, :type => :string } <<
+ line.captures << { :name => :bucket_owner, :type => :string } <<
+ { :name => :bucket, :type => :string } <<
{ :name => :timestamp, :type => :timestamp } <<
{ :name => :remote_ip, :type => :string } <<
- { :name => :requester, :type => :string } <<
+ { :name => :requester, :type => :string } <<
{ :name => :request_id, :type => :string } <<
{ :name => :operation, :type => :string } <<
{ :name => :key, :type => :nillable_string } <<
@@ -28,7 +28,7 @@ class AmazonS3 < Base
{ :name => :referer, :type => :referer } <<
{ :name => :user_agent, :type => :user_agent }
end
-
+
report do |analyze|
analyze.timespan
analyze.hourly_spread
@@ -39,33 +39,33 @@ class AmazonS3 < Base
analyze.frequency :category => :http_status, :title => 'HTTP status codes'
analyze.frequency :category => :error_code, :title => 'Error codes'
end
-
+
class Request < RequestLogAnalyzer::Request
-
+
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
-
+
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
# to speed up parsing.
def convert_timestamp(value, definition)
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
end
-
+
# Make sure that the string '-' is parsed as a nil value.
def convert_nillable_string(value, definition)
value == '-' ? nil : value
end
-
+
# Can be implemented in subclasses for improved categorizations
def convert_referer(value, definition)
value == '-' ? nil : value
end
-
+
# Can be implemented in subclasses for improved categorizations
def convert_user_agent(value, definition)
value == '-' ? nil : value
end
end
-
+
end
end
View
24 lib/request_log_analyzer/file_format/apache.rb
@@ -1,6 +1,6 @@
module RequestLogAnalyzer::FileFormat
-
- # The Apache file format is able to log Apache access.log files.
+
+ # The Apache file format is able to log Apache access.log files.
#
# The access.log can be configured in Apache to have many different formats. In theory, this
# FileFormat can handle any format, but it must be aware of the log formatting that is used
@@ -22,7 +22,7 @@ class Apache < Base
:referer => '%{Referer}i -> %U',
:agent => '%{User-agent}i'
}
-
+
# A hash that defines how the log format directives should be parsed.
LOG_DIRECTIVES = {
'%' => { :regexp => '%', :captures => [] },
@@ -62,7 +62,7 @@ def self.access_line_definition(format_string)
format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
-
+
if variable
# Check if we recognize the log directive
directive = LOG_DIRECTIVES[variable]
@@ -77,7 +77,7 @@ def self.access_line_definition(format_string)
end
end
end
-
+
# Return a new line definition object
return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
:captures => captures, :header => true, :footer => true)
@@ -105,32 +105,32 @@ def self.report_trackers(line_definition)
# Define a custom Request class for the Apache file format to speed up timestamp handling.
class Request < RequestLogAnalyzer::Request
-
+
def category
first(:path)
end
-
+
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
-
+
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
# to speed up parsing.
def convert_timestamp(value, definition)
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
end
-
+
# This function can be overridden to rewrite the path for better categorization in the
# reports.
def convert_path(value, definition)
value
end
-
- # This function can be overridden to simplify the user agent string for better
+
+ # This function can be overridden to simplify the user agent string for better
# categorization in the reports
def convert_user_agent(value, definition)
value # TODO
end
-
+
# Make sure that the string '-' is parsed as a nil value.
def convert_nillable_string(value, definition)
value == '-' ? nil : value
View
22 lib/request_log_analyzer/file_format/merb.rb
@@ -1,5 +1,5 @@
module RequestLogAnalyzer::FileFormat
-
+
# The Merb file format parses the request header with the timestamp, the params line
# with the most important request information and the durations line which contains
# the different request durations that can be used for analysis.
@@ -11,14 +11,14 @@ class Merb < Base
line.teaser = /Started request handling\:/
line.regexp = /Started request handling\:\ (.+)/
line.captures << { :name => :timestamp, :type => :timestamp }
- end
-
+ end
+
# ~ Params: {"action"=>"create", "controller"=>"session"}
# ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
line_definition :params do |line|
line.teaser = /Params\:\ /
line.regexp = /Params\:\ (\{.+\})/
- line.captures << { :name => :params, :type => :eval, :provides => {
+ line.captures << { :name => :params, :type => :eval, :provides => {
:namespace => :string, :controller => :string, :action => :string, :format => :string, :method => :string } }
end
@@ -31,32 +31,32 @@ class Merb < Base
:dispatch_time => :duration, :after_filters_time => :duration,
:before_filters_time => :duration, :action_time => :duration } }
end
-
- REQUEST_CATEGORIZER = Proc.new do |request|
+
+ REQUEST_CATEGORIZER = Proc.new do |request|
category = "#{request[:controller]}##{request[:action]}"
category = "#{request[:namespace]}::#{category}" if request[:namespace]
category = "#{category}.#{request[:format]}" if request[:format]
category
end
-
+
report do |analyze|
-
+
analyze.timespan
analyze.hourly_spread
analyze.frequency :category => REQUEST_CATEGORIZER, :amount => 20, :title => "Top 20 by hits"
analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
-
+
# analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
# analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'
# analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration'
end
class Request < RequestLogAnalyzer::Request
-
+
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
-
+
# Speed up timestamp conversion
def convert_timestamp(value, definition)
"#{value[26,4]}#{MONTHS[value[4,3]]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
View
8 lib/request_log_analyzer/file_format/rack.rb
@@ -1,11 +1,11 @@
module RequestLogAnalyzer::FileFormat
-
+
class Rack < Apache
-
+
def self.create(*args)
super(:rack, *args)
end
-
+
end
-
+
end
View
36 lib/request_log_analyzer/file_format/rails.rb
@@ -1,10 +1,10 @@
module RequestLogAnalyzer::FileFormat
-
+
class Rails < Base
-
+
# Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
line_definition :processing do |line|
- line.header = true # this line is the first log line for a request
+ line.header = true # this line is the first log line for a request
line.teaser = /Processing /
line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
line.captures << { :name => :controller, :type => :string } \
@@ -19,8 +19,8 @@ class Rails < Base
line_definition :cache_hit do |line|
line.regexp = /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter/
end
-
- # RuntimeError (Cannot destroy employee): /app/models/employee.rb:198:in `before_destroy'
+
+ # RuntimeError (Cannot destroy employee): /app/models/employee.rb:198:in `before_destroy'
line_definition :failed do |line|
line.footer = true
line.regexp = /((?:[A-Z]\w+\:\:)*[A-Z]\w+) \((.*)\)(?: on line #(\d+) of .+)?\:(.*)/
@@ -37,29 +37,29 @@ class Rails < Base
RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
# Rails > 2.1 completed line example
- # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
+ # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
- # The completed line uses a kind of hack to ensure that both old style logs and new style logs
+ # The completed line uses a kind of hack to ensure that both old style logs and new style logs
# are both parsed by the same regular expression. The format in Rails 2.2 was slightly changed,
# but the line contains exactly the same information.
line_definition :completed do |line|
-
+
line.footer = true
line.teaser = /Completed in /
line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
-
+
line.captures << { :name => :duration, :type => :duration, :unit => :sec } \
<< { :name => :view, :type => :duration, :unit => :sec } \
<< { :name => :db, :type => :duration, :unit => :sec } \
<< { :name => :status, :type => :integer } \
- << { :name => :url, :type => :string } # Old variant
-
+ << { :name => :url, :type => :string } # Old variant
+
line.captures << { :name => :duration, :type => :duration, :unit => :msec } \
<< { :name => :view, :type => :duration, :unit => :msec } \
<< { :name => :db, :type => :duration, :unit => :msec } \
<< { :name => :status, :type => :integer } \
- << { :name => :url, :type => :string } # 2.2 variant
+ << { :name => :url, :type => :string } # 2.2 variant
end
REQUEST_CATEGORIZER = Proc.new do |request|
@@ -69,26 +69,26 @@ class Rails < Base
report do |analyze|
analyze.timespan
analyze.hourly_spread
-
+
analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20
analyze.frequency :method, :title => 'HTTP methods'
analyze.frequency :status, :title => 'HTTP statuses returned'
analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
-
+
analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration", :line_type => :completed
analyze.duration :view, :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
analyze.duration :db, :category => REQUEST_CATEGORIZER, :title => "Database time", :line_type => :completed
-
- analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
+
+ analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
:if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
-
+
analyze.frequency :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
end
# Define a custom Request class for the Rails file format to speed up timestamp handling
# and to ensure that a format is always set.
class Request < RequestLogAnalyzer::Request
-
+
# Do not use DateTime.parse
def convert_timestamp(value, definition)
value.gsub(/[^0-9]/, '')[0...14].to_i
View
30 lib/request_log_analyzer/file_format/rails_development.rb
@@ -1,24 +1,24 @@
module RequestLogAnalyzer::FileFormat
-
+
# The RailsDevelopment FileFormat is an extention to the default Rails file format. It includes
# all lines of the normal Rails file format, but parses SQL queries and partial rendering lines
# as well.
class RailsDevelopment < Rails
-
+
# Parameters: {"action"=>"demo", "controller"=>"page"}
line_definition :parameters do |line|
line.teaser = /Parameters/
line.regexp = /\s+Parameters:\s+(\{.*\})/
line.captures << { :name => :params, :type => :eval }
end
-
+
# Rendered layouts/_footer (2.9ms)
line_definition :rendered do |line|
line.regexp = /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/
line.captures << { :name => :render_file, :type => :string } \
<< { :name => :render_duration, :type => :duration, :unit => :msec }
end
-
+
# User Load (0.4ms) SELECT * FROM `users` WHERE (`users`.`id` = 18205844) 
line_definition :query_executed do |line|
line.regexp = /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?([^\e]+) ?(?:\e\[0m)?/
@@ -27,29 +27,29 @@ class RailsDevelopment < Rails
<< { :name => :query_sql, :type => :sql }
end
- # [4;35;1mCACHE (0.0ms)[0m [0mSELECT * FROM `users` WHERE (`users`.`id` = 0) [0m
+ # [4;35;1mCACHE (0.0ms)[0m [0mSELECT * FROM `users` WHERE (`users`.`id` = 0) [0m
line_definition :query_cached do |line|
line.regexp = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?([^\e]+) ?(?:\e\[0m)?/
line.captures << { :name => :cached_duration, :type => :duration, :unit => :msec } \
<< { :name => :cached_sql, :type => :sql }
- end
+ end
# Define the reporting for the additional parsed lines
report(:append) do |analyze|
-
- analyze.duration :render_duration, :category => :render_file, :multiple_per_request => true,
+
+ analyze.duration :render_duration, :category => :render_file, :multiple_per_request => true,
:amount => 20, :title => 'Partial rendering duration'
-
- analyze.duration :query_duration, :category => :query_sql, :multiple_per_request => true,
+
+ analyze.duration :query_duration, :category => :query_sql, :multiple_per_request => true,
:amount => 20, :title => 'Query duration'
-
- end
-
+
+ end
+
# Add a converter method for the SQL fields the the Rails request class
class Request < Rails::Request
-
+
# Sanitizes SQL queries so that they can be grouped
- def convert_sql(sql, definition)
+ def convert_sql(sql, definition)
sql.gsub(/\b\d+\b/, ':int').gsub(/`([^`]+)`/, '\1').gsub(/'[^']*'/, ':string').rstrip
end
end
View
12 lib/request_log_analyzer/filter.rb
@@ -1,30 +1,30 @@
module RequestLogAnalyzer::Filter
-
+
# Filter class loader using const_missing
# This function will automatically load the class file based on the name of the class
def self.const_missing(const)
RequestLogAnalyzer::load_default_class_file(self, const)
end
-
+
# Base filter class used to filter input requests.
# All filters should interit from this base.
class Base