Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

cleanup status output

20071203084635-42483-ca38301345f4d8c74866b9d08e58f7376749395d.gz
  • Loading branch information...
commit 73b0b6e383c6bb80c77cea8040c72806c214455b 1 parent f21bc59
@copiousfreetime authored git-darcs-import committed
View
9 CHANGES
@@ -1,5 +1,10 @@
= Changelog
-== Version 0.2.3 - 2007-11-26
+== Version 0.3.0 - 2007-12-03
+
+* Added nice progession output
+* Improved rows parsing performance
+* Reorder directories
+* Prep for public release
+* Added configurable SMTP server for alerts
-* Initial public release
View
4 TODO
@@ -1 +1,3 @@
-* Add in progress meter to show how much of file is still left to parse.
+* variable commit points, so process can restart
+* signal handling to commit and finalize when killed
+* update documentation
View
4 lib/halog/application.rb
@@ -122,7 +122,7 @@ def error_version_help
def import_new_data(datastore)
if @options.input_file then
input_log_stream = (@options.input_file == "-") ? $stdin : File.open(@options.input_file)
- $stderr.puts "Reading input from #{@options.input_file}"
+ $stderr.puts "Input file : #{@options.input_file}"
datastore.import(input_log_stream,{:incremental => @options.incremental})
$stderr.puts datastore.perf_report
end
@@ -176,7 +176,7 @@ def run
merge_options
DataStore.open(@options.database) do |ds|
- $stderr.puts "Using #{@options.database} database."
+ $stderr.puts "Database : #{@options.database}"
import_new_data(ds)
run_report(ds)
end
View
8 lib/halog/datastore.rb
@@ -136,9 +136,13 @@ def perf_report
require 'stringio'
report = ::StringIO.new
name_width = @perf_info.keys.collect { |k| k.length }.max
+ header = ["Method".ljust(name_width), "Call count".rjust(15), "Time in call".rjust(15), "Calls per sec".rjust(15)].join(' ')
+
report.puts
- report.puts ["Method".ljust(name_width), "Call count".rjust(15), "Time in call".rjust(15), "Calls per sec".rjust(15)].join(' ')
- report.puts "-" * (name_width + 45 + 3)
+ report.puts "PERFORMANCE STATS".center(header.size)
+ report.puts header
+ report.puts "-" * header.size
+
@perf_info.keys.sort.each do |stat|
values = @perf_info[stat]
avg = values['count'].to_f / values['time'].to_f
View
81 lib/halog/log_parser.rb
@@ -23,6 +23,7 @@ def initialize()
@entry_count = 0
@error_count = 0
@parse_time = 0
+ @start_time = 0
end
# parses the io stream yielding each valid LogEntry that is encountered. Invalid lines
@@ -31,19 +32,10 @@ def parse(io,options = {})
io = advance_io(io,options)
@starting_offset = io.pos
total_bytes = io.stat.size - @starting_offset
- start_time = Time.now
-
+ @start_time = Time.now
+
$stderr.puts
- $stderr.puts [
- "Lines".rjust(10),
- "Row rate".rjust(10),
- "Byte rate".rjust(12),
- "Progress".rjust(20),
- "% Done".rjust(8),
- "Timer".rjust(10),
- "Time left".rjust(10)
- ].join(' ')
- $stderr.puts '-' * (10 + 10 + 12 + 20 + 8 + 10 + 10 + 6)
+ $stderr.puts status_header
io.each do |line|
@@ -71,36 +63,55 @@ def parse(io,options = {})
end
if @entry_count % REPORT_EVERY == 0 then
- current_pos = io.pos
- completed_bytes = current_pos - @starting_offset
- elapsed_time = Time.now - start_time
- bytes_left = total_bytes - completed_bytes
- byte_rate = completed_bytes.to_f / elapsed_time
- time_left = bytes_left / byte_rate
- rps = @entry_count / elapsed_time
- bps = completed_bytes / elapsed_time
- percent_complete = (completed_bytes.to_f / total_bytes) * 100.0
-
- status = [
- "#{@entry_count}".rjust(10),
- "#{"%.0f" % rps} rps".rjust(10),
- "#{num_to_bytes(bps)}/s".rjust(12),
- "#{num_to_bytes(completed_bytes)}/#{num_to_bytes(total_bytes)}".rjust(20),
- "#{"%.2f" % percent_complete}%".rjust(8),
- hms_from_seconds(elapsed_time).rjust(10),
- hms_from_seconds(time_left).rjust(10)
- ]
-
- $stderr.print "#{status.join(' ')}\r"
- $stderr.flush
+ $stderr.print "#{status_output(io.pos,total_bytes)}\r"
end
end # io.each
+ $stderr.puts status_output(io.pos,total_bytes)
$stderr.puts
- $stderr.puts "Done. parsed lines: #{@entry_count}"
+
return self
end
+ def status_header
+ output = StringIO.new
+ header = [
+ "Lines".rjust(10),
+ "Row rate".rjust(10),
+ "Byte rate".rjust(12),
+ "Progress".rjust(20),
+ "% Done".rjust(8),
+ "Timer".rjust(10),
+ "Time left".rjust(10)
+ ].join(' ')
+ output.puts header
+ output.print "-" * header.size
+ output.string
+ end
+
+ def status_output(current_pos,total_bytes)
+ completed_bytes = current_pos - @starting_offset
+ elapsed_time = Time.now - @start_time
+ bytes_left = total_bytes - completed_bytes
+ byte_rate = completed_bytes.to_f / elapsed_time
+ time_left = bytes_left / byte_rate
+ rps = @entry_count / elapsed_time
+ bps = completed_bytes / elapsed_time
+ percent_complete = (completed_bytes.to_f / total_bytes) * 100.0
+
+ status = [
+ "#{@entry_count}".rjust(10),
+ "#{"%.0f" % rps} rps".rjust(10),
+ "#{num_to_bytes(bps)}/s".rjust(12),
+ "#{num_to_bytes(completed_bytes)}/#{num_to_bytes(total_bytes)}".rjust(20),
+ "#{"%.2f" % percent_complete}%".rjust(8),
+ hms_from_seconds(elapsed_time).rjust(10),
+ hms_from_seconds(time_left).rjust(10)
+ ]
+
+ status.join(' ')
+ end
+
# advance the IO forward if it looks like this logfile has already been parsed.
# advancing the IO only works on actual files. IO that comes from STDIN
def advance_io(io,options)
Please sign in to comment.
Something went wrong with that request. Please try again.