Skip to content
This repository
Browse code

+ more efficient log file checking

  • Loading branch information...
commit c990b8d03667d30af26388ad0b07ee7714266100 1 parent cf66baa
Florian R. Hanke authored December 15, 2011
75  statistics/lib/picky-statistics/statistics/logfile_reader.rb
@@ -4,16 +4,20 @@
4 4
 module Statistics
5 5
 
6 6
   class LogfileReader
7  
-    
8  
-    attr_reader :last_offset
  7
+
  8
+    attr_reader :last_offset # in bytes
9 9
     attr_writer :path
10 10
 
  11
+    def exclaim text
  12
+      puts text
  13
+    end
  14
+
11 15
     #
12 16
     #
13 17
     def initialize path
14 18
       @path = File.expand_path path
15 19
       check_file
16  
-      
  20
+
17 21
       @counts = {}
18 22
       initialize_totals
19 23
       initialize_full_totals
@@ -22,19 +26,19 @@ def initialize path
22 26
     end
23 27
     def check_file
24 28
       if File.exists? @path
25  
-        puts "Logfile #{@path} found."
  29
+        exclaim "Logfile #{@path} found."
26 30
       else
27 31
         raise "Log file #{@path} not found."
28 32
       end
29 33
     end
30  
-    
  34
+
31 35
     def full
32 36
       @counts[:full] ||= {}
33 37
     end
34 38
     def live
35 39
       @counts[:live] ||= {}
36 40
     end
37  
-    
  41
+
38 42
     def initialize_totals
39 43
       full[:total] = Count.new "^>|"
40 44
       live[:total] = Count.new "^\\.|"
@@ -48,18 +52,18 @@ def initialize_full_totals
48 52
       full[:totals][:'4+']     = Count.new("^>|.*|       [4-9]|", "^>|.*|      [1-9][0-9]|") # 4-9, 10+, but less than 100
49 53
       full[:totals][:'100+']   = Count.new "^>|.*|.*|     [1-9][0-9][0-9]|"
50 54
       full[:totals][:'1000+']  = Count.new "^>|.*|.*|....[0-9]...|"
51  
-                                       
  55
+
52 56
       full[:totals][:cloud]     = Count.new("^>|.*|[1-9].|",                      # allocs 10+
53 57
                                             "^>|.*|.*|......[1-9].|....|.[2-9]|") # allocs 2-9, more than 10 results
54 58
     end
55 59
     def initialize_live_totals
56  
-      
  60
+
57 61
     end
58 62
     def initialize_other
59 63
       full[:quick]             = Count.new "^>|.\\+\\?|0\\.00....|"
60 64
       full[:long_running]      = Count.new "^>|.\\+\\?|0\\.[1-9].....|"
61 65
       full[:very_long_running] = Count.new "^>|.\\+\\?|[1-9]\\.......|"
62  
-      
  66
+
63 67
       full[:offset]            = Count.new("^>|.*|[ 1-9][ 0-9][0-9][0-9]|", # offset 10+
64 68
                                            "^>|.*|   [1-9]")                # offset 1-9
65 69
     end
@@ -67,43 +71,46 @@ def initialize_other
67 71
     #
68 72
     #
69 73
     def since_last
70  
-      @last_offset ||= 1
71  
-      
  74
+      @last_offset ||= 0
  75
+
72 76
       log_offset = @last_offset
73 77
       start_time = Time.now
74  
-      
  78
+
75 79
       with_temp_file(@last_offset) do |statistics|
76 80
         calculate_last_offset_from statistics
77  
-        
  81
+
78 82
         full[:total].add_from statistics
79 83
         live[:total].add_from statistics
80  
-        
  84
+
81 85
         full[:totals][0].add_from statistics
82 86
         full[:totals][1].add_from statistics
83 87
         full[:totals][2].add_from statistics
84 88
         full[:totals][3].add_from statistics
85  
-        
  89
+
86 90
         full[:totals][:'4+'].add_from statistics
87 91
         full[:totals][:cloud].add_from statistics
88  
-        
  92
+
89 93
         full[:totals][:'100+'].add_from statistics
90 94
         full[:totals][:'1000+'].add_from statistics
91  
-        
  95
+
92 96
         full[:quick].add_from statistics
93 97
         full[:long_running].add_from statistics
94 98
         full[:very_long_running].add_from statistics
95  
-        
  99
+
96 100
         full[:offset].add_from statistics
97 101
       end
98  
-      
  102
+
99 103
       duration = Time.now - start_time
100  
-      puts "Parsed log from line #{log_offset} in #{duration}s"
101  
-      
  104
+      exclaim "Parsed log from byte #{log_offset} in #{duration}s"
  105
+
102 106
       @counts
103 107
     end
104  
-    
  108
+
105 109
     def calculate_last_offset_from statistics
106  
-      @last_offset += `wc -l #{statistics}`.to_i
  110
+      @last_offset += last_offset_from statistics
  111
+    end
  112
+    def last_offset_from statistics
  113
+      `wc #{statistics}`.split(/\s+/)[3].to_i
107 114
     end
108 115
 
109 116
     #
@@ -114,19 +121,23 @@ def reset_from time
114 121
       end
115 122
       @counts
116 123
     end
117  
-    
  124
+
118 125
     # Use the offset to speed up statistics gathering.
119 126
     #
120 127
     def with_temp_file offset = 0
121  
-      temp_file = Tempfile.new 'picky'
122  
-      
123  
-      temp_path = temp_file.path
124  
-      `tail -n +#{offset} #{@path} > #{temp_path}`
125  
-      yield temp_path
126  
-      
127  
-      temp_file.delete
  128
+      # Quickly return if no logs have been written since the last time.
  129
+      #
  130
+      return if last_offset_from(@path) <= last_offset
  131
+
  132
+      Tempfile.open 'picky' do |temp_file|
  133
+
  134
+        temp_path = temp_file.path
  135
+        `tail -c +#{offset} #{@path} > #{temp_path}`
  136
+        yield temp_path
  137
+
  138
+      end
128 139
     end
129  
-  
  140
+
130 141
   end
131 142
 
132 143
 end
2  statistics/random_queries.rb
@@ -2,7 +2,7 @@
2 2
 queries = %w|t te tes test testi tests|
3 3
 ids     = [0, 20]
4 4
 
5  
-10_000.times do
  5
+1000.times do
6 6
   offset = offsets[rand(offsets.size)]
7 7
   query  = queries[rand(queries.size)]
8 8
   id     = ids[rand(ids.size)]
53  statistics/spec/lib/picky-statistics/statistics/logfile_reader_spec.rb
... ...
@@ -1,44 +1,47 @@
1 1
 require 'spec_helper'
2 2
 
3 3
 describe Statistics::LogfileReader do
4  
-  
5  
-  before(:each) do
6  
-    @reader = Statistics::LogfileReader.new 'spec/data/search.log'
  4
+
  5
+  let(:reader) do
  6
+    reader = Statistics::LogfileReader.allocate
  7
+    reader.stub! :exclaim
  8
+    reader.send :initialize, 'spec/data/search.log'
  9
+    reader
7 10
   end
8  
-  
  11
+
9 12
   describe 'full' do
10 13
     it 'is set to have the right keys' do
11  
-      @reader.full.should have_key(:total)
12  
-      @reader.full.should have_key(:quick)
13  
-      @reader.full.should have_key(:long_running)
14  
-      @reader.full.should have_key(:very_long_running)
15  
-      @reader.full.should have_key(:offset)
16  
-      
17  
-      @reader.full[:totals].should have_key(0)
18  
-      @reader.full[:totals].should have_key(1)
19  
-      @reader.full[:totals].should have_key(2)
20  
-      @reader.full[:totals].should have_key(3)
21  
-      @reader.full[:totals].should have_key(:'4+')
22  
-      @reader.full[:totals].should have_key(:'100+')
23  
-      @reader.full[:totals].should have_key(:'1000+')
24  
-      @reader.full[:totals].should have_key(:cloud)
  14
+      reader.full.should have_key(:total)
  15
+      reader.full.should have_key(:quick)
  16
+      reader.full.should have_key(:long_running)
  17
+      reader.full.should have_key(:very_long_running)
  18
+      reader.full.should have_key(:offset)
  19
+
  20
+      reader.full[:totals].should have_key(0)
  21
+      reader.full[:totals].should have_key(1)
  22
+      reader.full[:totals].should have_key(2)
  23
+      reader.full[:totals].should have_key(3)
  24
+      reader.full[:totals].should have_key(:'4+')
  25
+      reader.full[:totals].should have_key(:'100+')
  26
+      reader.full[:totals].should have_key(:'1000+')
  27
+      reader.full[:totals].should have_key(:cloud)
25 28
     end
26 29
   end
27 30
   describe 'live' do
28 31
     it 'is set to a specific Hash at first' do
29  
-      @reader.live.should have_key(:total)
  32
+      reader.live.should have_key(:total)
30 33
     end
31 34
   end
32  
-  
  35
+
33 36
   describe 'since_last' do
34 37
     it 'returns a specific JSON result' do
35  
-      @reader.since_last.to_json.should == '{"full":{"total":"22","totals":{"0":"1","1":"1","2":"1","3":"1","4+":"6","100+":"2","1000+":"10","cloud":"3"},"quick":"16","long_running":"2","very_long_running":"2","offset":"5"},"live":{"total":"1"}}'
  38
+      reader.since_last.to_json.should == '{"full":{"total":"22","totals":{"0":"1","1":"1","2":"1","3":"1","4+":"6","100+":"2","1000+":"10","cloud":"3"},"quick":"16","long_running":"2","very_long_running":"2","offset":"5"},"live":{"total":"1"}}'
36 39
     end
37 40
     it 'is at a specific line after reading the log file' do
38  
-      @reader.since_last
39  
-      
40  
-      @reader.last_offset.should == 23
  41
+      reader.since_last
  42
+
  43
+      reader.last_offset.should == 2299
41 44
     end
42 45
   end
43  
-  
  46
+
44 47
 end

0 notes on commit c990b8d

Please sign in to comment.
Something went wrong with that request. Please try again.