Permalink
Browse files

+ more efficient log file checking

  • Loading branch information...
1 parent cf66baa commit c990b8d03667d30af26388ad0b07ee7714266100 @floere committed Dec 15, 2011
@@ -4,16 +4,20 @@
module Statistics
class LogfileReader
-
- attr_reader :last_offset
+
+ attr_reader :last_offset # in bytes
attr_writer :path
+ def exclaim text
+ puts text
+ end
+
#
#
def initialize path
@path = File.expand_path path
check_file
-
+
@counts = {}
initialize_totals
initialize_full_totals
@@ -22,19 +26,19 @@ def initialize path
end
def check_file
if File.exists? @path
- puts "Logfile #{@path} found."
+ exclaim "Logfile #{@path} found."
else
raise "Log file #{@path} not found."
end
end
-
+
def full
@counts[:full] ||= {}
end
def live
@counts[:live] ||= {}
end
-
+
def initialize_totals
full[:total] = Count.new "^>|"
live[:total] = Count.new "^\\.|"
@@ -48,62 +52,65 @@ def initialize_full_totals
full[:totals][:'4+'] = Count.new("^>|.*| [4-9]|", "^>|.*| [1-9][0-9]|") # 4-9, 10+, but less than 100
full[:totals][:'100+'] = Count.new "^>|.*|.*| [1-9][0-9][0-9]|"
full[:totals][:'1000+'] = Count.new "^>|.*|.*|....[0-9]...|"
-
+
full[:totals][:cloud] = Count.new("^>|.*|[1-9].|", # allocs 10+
"^>|.*|.*|......[1-9].|....|.[2-9]|") # allocs 2-9, more than 10 results
end
def initialize_live_totals
-
+
end
def initialize_other
full[:quick] = Count.new "^>|.\\+\\?|0\\.00....|"
full[:long_running] = Count.new "^>|.\\+\\?|0\\.[1-9].....|"
full[:very_long_running] = Count.new "^>|.\\+\\?|[1-9]\\.......|"
-
+
full[:offset] = Count.new("^>|.*|[ 1-9][ 0-9][0-9][0-9]|", # offset 10+
"^>|.*| [1-9]") # offset 1-9
end
#
#
def since_last
- @last_offset ||= 1
-
+ @last_offset ||= 0
+
log_offset = @last_offset
start_time = Time.now
-
+
with_temp_file(@last_offset) do |statistics|
calculate_last_offset_from statistics
-
+
full[:total].add_from statistics
live[:total].add_from statistics
-
+
full[:totals][0].add_from statistics
full[:totals][1].add_from statistics
full[:totals][2].add_from statistics
full[:totals][3].add_from statistics
-
+
full[:totals][:'4+'].add_from statistics
full[:totals][:cloud].add_from statistics
-
+
full[:totals][:'100+'].add_from statistics
full[:totals][:'1000+'].add_from statistics
-
+
full[:quick].add_from statistics
full[:long_running].add_from statistics
full[:very_long_running].add_from statistics
-
+
full[:offset].add_from statistics
end
-
+
duration = Time.now - start_time
- puts "Parsed log from line #{log_offset} in #{duration}s"
-
+ exclaim "Parsed log from byte #{log_offset} in #{duration}s"
+
@counts
end
-
+
def calculate_last_offset_from statistics
- @last_offset += `wc -l #{statistics}`.to_i
+ @last_offset += last_offset_from statistics
+ end
+ def last_offset_from statistics
+ `wc #{statistics}`.split(/\s+/)[3].to_i
end
#
@@ -114,19 +121,23 @@ def reset_from time
end
@counts
end
-
+
# Use the offset to speed up statistics gathering.
#
def with_temp_file offset = 0
- temp_file = Tempfile.new 'picky'
-
- temp_path = temp_file.path
- `tail -n +#{offset} #{@path} > #{temp_path}`
- yield temp_path
-
- temp_file.delete
+ # Quickly return if no logs have been written since the last time.
+ #
+ return if last_offset_from(@path) <= last_offset
+
+ Tempfile.open 'picky' do |temp_file|
+
+ temp_path = temp_file.path
+ `tail -c +#{offset} #{@path} > #{temp_path}`
+ yield temp_path
+
+ end
end
-
+
end
end
@@ -2,7 +2,7 @@
queries = %w|t te tes test testi tests|
ids = [0, 20]
-10_000.times do
+1000.times do
offset = offsets[rand(offsets.size)]
query = queries[rand(queries.size)]
id = ids[rand(ids.size)]
@@ -1,44 +1,47 @@
require 'spec_helper'
describe Statistics::LogfileReader do
-
- before(:each) do
- @reader = Statistics::LogfileReader.new 'spec/data/search.log'
+
+ let(:reader) do
+ reader = Statistics::LogfileReader.allocate
+ reader.stub! :exclaim
+ reader.send :initialize, 'spec/data/search.log'
+ reader
end
-
+
describe 'full' do
it 'is set to have the right keys' do
- @reader.full.should have_key(:total)
- @reader.full.should have_key(:quick)
- @reader.full.should have_key(:long_running)
- @reader.full.should have_key(:very_long_running)
- @reader.full.should have_key(:offset)
-
- @reader.full[:totals].should have_key(0)
- @reader.full[:totals].should have_key(1)
- @reader.full[:totals].should have_key(2)
- @reader.full[:totals].should have_key(3)
- @reader.full[:totals].should have_key(:'4+')
- @reader.full[:totals].should have_key(:'100+')
- @reader.full[:totals].should have_key(:'1000+')
- @reader.full[:totals].should have_key(:cloud)
+ reader.full.should have_key(:total)
+ reader.full.should have_key(:quick)
+ reader.full.should have_key(:long_running)
+ reader.full.should have_key(:very_long_running)
+ reader.full.should have_key(:offset)
+
+ reader.full[:totals].should have_key(0)
+ reader.full[:totals].should have_key(1)
+ reader.full[:totals].should have_key(2)
+ reader.full[:totals].should have_key(3)
+ reader.full[:totals].should have_key(:'4+')
+ reader.full[:totals].should have_key(:'100+')
+ reader.full[:totals].should have_key(:'1000+')
+ reader.full[:totals].should have_key(:cloud)
end
end
describe 'live' do
it 'is set to a specific Hash at first' do
- @reader.live.should have_key(:total)
+ reader.live.should have_key(:total)
end
end
-
+
describe 'since_last' do
it 'returns a specific JSON result' do
- @reader.since_last.to_json.should == '{"full":{"total":"22","totals":{"0":"1","1":"1","2":"1","3":"1","4+":"6","100+":"2","1000+":"10","cloud":"3"},"quick":"16","long_running":"2","very_long_running":"2","offset":"5"},"live":{"total":"1"}}'
+ reader.since_last.to_json.should == '{"full":{"total":"22","totals":{"0":"1","1":"1","2":"1","3":"1","4+":"6","100+":"2","1000+":"10","cloud":"3"},"quick":"16","long_running":"2","very_long_running":"2","offset":"5"},"live":{"total":"1"}}'
end
it 'is at a specific line after reading the log file' do
- @reader.since_last
-
- @reader.last_offset.should == 23
+ reader.since_last
+
+ reader.last_offset.should == 2299
end
end
-
+
end

0 comments on commit c990b8d

Please sign in to comment.