diff --git a/lib/ralf/clf_translator.rb b/lib/ralf/clf_translator.rb index f807c0f..b6596ed 100644 --- a/lib/ralf/clf_translator.rb +++ b/lib/ralf/clf_translator.rb @@ -5,8 +5,15 @@ class Ralf::ClfTranslator attr :line attr_reader :owner, :bucket, :timestamp, :remote_ip, :request_id, :operation, :key, :request_uri, :http_status, :s3_error_code, :bytes_sent, :object_size, :total_time_in_ms, :turn_around_time_in_ms, :referrer, :user_agent, :request_version_id, :duration + attr_reader :options - def initialize(line) + # options: + # :fix_partial_content => false (default) + # If request is '206 Partial Content' estimate the actual bytes when apparent bandwidth has exceeded 2Mbit/sec. + # S3 caches content to edge servers with a burst which never reaches the client + + def initialize(line, options = {}) + @options = options @error = false @line = line @translate_successfull = translate @@ -30,8 +37,7 @@ def translate if line =~ AMAZON_LOG_FORMAT @owner, @bucket, @timestamp, @remote_ip, @requester, @request_id, @operation, @key, @request_uri, @http_status, @s3_error_code, @bytes_sent, @object_size, @total_time_in_ms, @turn_around_time_in_ms, @referrer, @user_agent, @request_version_id = $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 - # If request is '206 Partial Content' estimate the actual bytes when apparent bandwidth has exceeded 2Mbit/sec. - if 206 == http_status.to_i && ((bytes_sent.to_i*8)/total_time_in_ms.to_i > 2000) + if options[:fix_partial_content] && 206 == http_status.to_i && ((bytes_sent.to_i*8)/total_time_in_ms.to_i > 2000) @bytes_sent = [ 128 * 1024 + 3 * total_time_in_ms.to_i, bytes_sent.to_i ].min # 128 K buffer + 3 bytes/msec = 3 kbytes/sec = 24 kbit/sec end @duration = (total_time_in_ms.to_i/1000.0).round diff --git a/spec/ralf/clf_translator_spec.rb b/spec/ralf/clf_translator_spec.rb index 6267d3a..deae32f 100644 --- a/spec/ralf/clf_translator_spec.rb +++ b/spec/ralf/clf_translator_spec.rb @@ -26,11 +26,23 @@ $stderr.string.should eql("# ERROR: #{invalid_line}\n") end - it "should add rounded total_time in seconds" do - $stderr = StringIO.new - input_line = '2010-10-02-19-15-45-6879098C3140BE9D:2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [02/Oct/2010:18:29:16 +0000] 82.168.113.55 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 4F911681022807C6 REST.GET.OBJECT 10028050/2010-09-26-1830.mp3 "GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4194304 17537676 1600 12 "-" "VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team" -' - Ralf::ClfTranslator.new(input_line).to_s.should eql("82.168.113.55 - 2cf7e6b063 [02/Oct/2010:18:29:16 +0000] \"GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1\" 206 4194304 \"-\" \"VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team\" 2") - $stderr.string.should be_empty + describe "#option{:fix_partial_content}" do + describe "set to false (default)" do + it "should add rounded total_time in seconds" do + $stderr = StringIO.new + input_line = '2010-10-02-19-15-45-6879098C3140BE9D:2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [02/Oct/2010:18:29:16 +0000] 82.168.113.55 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 4F911681022807C6 REST.GET.OBJECT 10028050/2010-09-26-1830.mp3 "GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4194304 17537676 1600 12 "-" "VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team" -' + Ralf::ClfTranslator.new(input_line).to_s.should eql("82.168.113.55 - 2cf7e6b063 [02/Oct/2010:18:29:16 +0000] \"GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1\" 206 4194304 \"-\" \"VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team\" 2") + $stderr.string.should be_empty + end + end + describe "set to true" do + it "should add rounded total_time in seconds" do + $stderr = StringIO.new + input_line = '2010-10-02-19-15-45-6879098C3140BE9D:2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [02/Oct/2010:18:29:16 +0000] 82.168.113.55 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 4F911681022807C6 REST.GET.OBJECT 10028050/2010-09-26-1830.mp3 "GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4194304 17537676 1600 12 "-" "VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team" -' + Ralf::ClfTranslator.new(input_line, :fix_partial_content => true).to_s.should eql("82.168.113.55 - 2cf7e6b063 [02/Oct/2010:18:29:16 +0000] \"GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1\" 206 135872 \"-\" \"VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team\" 2") + $stderr.string.should be_empty + end + end end [