Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Put items, retry on error, handle redirects

  • Loading branch information...
commit d906c0fac37008933a2552bbafd4b79f28deab2c 1 parent 138c1ff
@jweiss jweiss authored
View
11 Rakefile
@@ -0,0 +1,11 @@
+require 'rake'
+require 'rake/testtask'
+
+task :default => [:test]
+
+Rake::TestTask.new(:test) do |t|
+ t.libs << 'test'
+ t.pattern = "test/**/*_test.rb"
+ t.verbose = true
+end
+
View
106 benchmark/right_aws.rb
@@ -0,0 +1,106 @@
+require File.dirname(__FILE__) + '/../happening'
+
+require 'benchmark'
+require 'right_aws'
+
+AWS_ACCESS_KEY_ID = ENV['AWS_ACCESS_KEY_ID'] or raise "please set AWS_ACCESS_KEY_ID='your-key'"
+AWS_SECRET_ACCESS_KEY = ENV['AWS_SECRET_ACCESS_KEY'] or raise "please set AWS_SECRET_ACCESS_KEY='your-scret'"
+
+BUCKET = 'happening-benchmark'
+FILE = 'the_file_name'
+PROTOCOL = 'https'
+
+COUNT = 100
+CONTENT = File.read('/tmp/VzLinuxUG.pdf')
+
+command = ARGV.first || 'get'
+
+puts "running command: #{command}"
+
+if command == 'get'
+ Benchmark.bm(7) do |x|
+ x.report("RightAWS - Get an item") do
+ count = COUNT
+ s3 = RightAws::S3Interface.new(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, :protocol => PROTOCOL)
+ count.times do |i|
+ s3.get_object(BUCKET, FILE)
+ print '.'; $stdout.flush
+ end
+ end
+
+ puts ""
+ x.report("Happening - Get an item") do
+ puts ""
+ count = COUNT
+ on_success = Proc.new do |http|
+ print '.'; $stdout.flush
+ count = count - 1
+ EM.stop if count <= 0
+ end
+
+ on_error = Proc.new do |http|
+ puts "Status: #{http.response_header.status}"
+ puts "Header: #{http.response_header.inspect}"
+ puts "Content:"
+ puts http.response.inspect + "\n"
+ count = count - 1
+ EM.stop if count <= 0
+ end
+
+ EM.run do
+ count.times do |i|
+ item = Happening::S3::Item.new(BUCKET, FILE, :protocol => PROTOCOL, :on_success => on_success, :on_error => on_error)
+ item.get
+ end
+ end
+ end
+ end
+
+elsif command == 'put'
+ Benchmark.bm(7) do |x|
+ x.report("RightAWS - Put an item") do
+ count = COUNT
+ s3 = RightAws::S3Interface.new(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, :protocol => PROTOCOL)
+ count.times do |i|
+ s3.put(BUCKET, "upload_test_right_aws_#{i}", CONTENT)
+ print '.'; $stdout.flush
+ end
+ end
+
+ puts ""
+ x.report("Happening - Put an item") do
+ puts ""
+ count = COUNT
+ on_success = Proc.new do |http|
+ #puts "Success"
+ puts "Status: #{http.response_header.status}" unless http.response_header.status == 200
+ #puts "Header: #{http.response_header.inspect}"
+ #puts "Content:"
+ #puts http.response.inspect + "\n"
+ print '.'; $stdout.flush
+ count = count - 1
+ EM.stop if count <= 0
+ end
+
+ on_error = Proc.new do |http|
+ puts "Error"
+ puts "Status: #{http.response_header.status}"
+ puts "Header: #{http.response_header.inspect}"
+ puts "Content:"
+ puts http.response.inspect + "\n"
+ count = count - 1
+ EM.stop if count <= 0
+ end
+
+ EM.run do
+ count.times do |i|
+ item = Happening::S3::Item.new(BUCKET, "upload_test_happening_#{i}", :protocol => PROTOCOL, :on_success => on_success, :on_error => on_error, :aws_access_key_id => AWS_ACCESS_KEY_ID, :aws_secret_access_key => AWS_SECRET_ACCESS_KEY)
+ item.put(CONTENT)
+ end
+ end
+ end
+ end
+
+else
+ puts "unknown command: #{command}"
+end
View
79 lib/s3/item.rb
@@ -17,9 +17,10 @@ def initialize(bucket, aws_id, options = {})
:server => 's3.amazonaws.com',
:protocol => 'https',
:aws_access_key_id => nil,
- :aws_secret_access_key => nil
+ :aws_secret_access_key => nil,
+ :retry_count => 4
}.update(options.symbolize_keys)
- options.assert_valid_keys(:timeout, :on_success, :on_error, :server, :protocol, :aws_access_key_id, :aws_secret_access_key)
+ options.assert_valid_keys(:timeout, :on_success, :on_error, :server, :protocol, :aws_access_key_id, :aws_secret_access_key, :retry_count)
@aws_id = aws_id.to_s
@bucket = bucket.to_s
@@ -27,19 +28,22 @@ def initialize(bucket, aws_id, options = {})
end
def get
- puts "Starting EM-HTTP request to #{url}"
-
headers = needs_to_sign? ? aws.sign("GET", path) : {}
-
+
http = http_class.new(url).get(:timeout => options[:timeout], :head => headers)
- http.errback {
- options[:on_error].call(http) if options[:on_error].respond_to?(:call)
- }
+ http.errback { error_callback(http) }
+ http.callback { success_callback(http) }
+ nil
+ end
+
+ def put(data)
+ headers = needs_to_sign? ? aws.sign("PUT", path, {'url' => path}) : {}
+ http = http_class.new(url).put(:timeout => options[:timeout], :head => headers, :body => data)
- http.callback {
- options[:on_success].call(http) if options[:on_success].respond_to?(:call)
- }
+ http.errback { error_callback(http) }
+ http.callback { success_callback(http, data) }
+ nil
end
def url
@@ -60,6 +64,59 @@ def path(with_bucket=true)
protected
+ def error_callback(http)
+ options[:on_error].call(http) if options[:on_error].respond_to?(:call)
+ end
+
+ def success_callback(http, data=nil)
+ case http.response_header.status
+ when 0, 400, 401, 404, 403, 409, 411, 412, 416, 500, 503
+ puts "retrying after: status #{http.response_header.status rescue ''}"
+ handle_retry(data)
+ when 300, 301, 303, 304, 307
+ puts "being redirected_to: #{http.response_header['LOCATION'] rescue ''}"
+ handle_redirect(http.response_header['LOCATION'], data)
+ else
+ options[:on_success].call(http) if options[:on_success].respond_to?(:call)
+ end
+ end
+
+ def handle_retry(data)
+ if options[:retry_count] > 0
+ if data
+ self.class.new(bucket, aws_id, options.update(:retry_count => options[:retry_count] - 1 )).put(data)
+ else
+ self.class.new(bucket, aws_id, options.update(:retry_count => options[:retry_count] - 1 )).get
+ end
+ else
+ puts "Re-tried too often - giving up"
+ end
+ end
+
+ def handle_redirect(location, data)
+ new_server, new_path = extract_location(location)
+
+ if data
+ self.class.new(bucket, aws_id, options.update(:server => new_server)).put(data)
+ else
+ self.class.new(bucket, aws_id, options.update(:server => new_server)).get
+ end
+ end
+
+ def extract_location(location)
+ uri = URI.parse(location)
+ if match = uri.host.match(/\A#{bucket}\.(.*)/)
+ server = match[1]
+ path = uri.path
+ elsif match = uri.path.match(/\A\/#{bucket}\/(.*)/)
+ server = uri.host
+ path = match[1]
+ else
+ raise "being redirected to an not understood place: #{location}"
+ end
+ return server, path.sub(/^\//, '')
+ end
+
def needs_to_sign?
options[:aws_access_key_id].present?
end
View
108 test/s3/item_test.rb
@@ -94,12 +94,118 @@ class ItemTest < Test::Unit::TestCase
end
end
+
+ should "retry on error" do
+ EventMachine::MockHttpRequest.register('https://bucket.s3.amazonaws.com:443/the-key', :get, {}, error_response(400))
+
+ @item = Happening::S3::Item.new('bucket', 'the-key')
+ run_in_em_loop do
+ @item.get
+
+ EM.add_timer(1) {
+ EM.stop_event_loop
+ assert_equal 5, EventMachine::MockHttpRequest.count('https://bucket.s3.amazonaws.com:443/the-key', :get, {})
+ }
+
+ end
+ end
+
+ should "handle re-direct" do
+ EventMachine::MockHttpRequest.register('https://bucket.s3.amazonaws.com:443/the-key', :get, {}, redirect_response('https://bucket.s3-external-3.amazonaws.com/the-key'))
+ EventMachine::MockHttpRequest.register('https://bucket.s3-external-3.amazonaws.com:443/the-key', :get, {}, fake_response('hy there'))
+
+ @item = Happening::S3::Item.new('bucket', 'the-key')
+ run_in_em_loop do
+ @item.get
+
+ EM.add_timer(1) {
+ EM.stop_event_loop
+ assert_equal 1, EventMachine::MockHttpRequest.count('https://bucket.s3.amazonaws.com:443/the-key', :get, {})
+ assert_equal 1, EventMachine::MockHttpRequest.count('https://bucket.s3-external-3.amazonaws.com:443/the-key', :get, {})
+ }
+
+ end
+ end
end
context "when saving an item" do
+ setup do
+ @time = "Thu, 25 Feb 2010 10:00:00 GMT"
+ Time.stubs(:now).returns(stub(:httpdate => @time, :to_i => 99, :usec => 88))
+ end
+
should "post to the desired location" do
- raise 'implement'
+ EventMachine::MockHttpRequest.register('https://bucket.s3.amazonaws.com:443/the-key', :put, {
+ "Authorization"=>"AWS abc:lZMKxGDKcQ1PH8yjbpyN7o2sPWg=",
+ 'date' => @time,
+ 'url' => "/bucket/the-key"}, fake_response("data-here"))
+
+ @item = Happening::S3::Item.new('bucket', 'the-key', :aws_access_key_id => 'abc', :aws_secret_access_key => '123')
+ run_in_em_loop do
+ @item.put('content')
+
+ EM.add_timer(1) {
+ EM.stop_event_loop
+ assert_equal 1, EventMachine::MockHttpRequest.count('https://bucket.s3.amazonaws.com:443/the-key', :put, {
+ "Authorization"=>"AWS abc:lZMKxGDKcQ1PH8yjbpyN7o2sPWg=",
+ 'date' => @time,
+ 'url' => "/bucket/the-key"})
+ }
+
+ end
+ end
+
+ should "re-post to a new location" do
+ EventMachine::MockHttpRequest.register('https://bucket.s3.amazonaws.com:443/the-key', :put, {
+ "Authorization"=>"AWS abc:lZMKxGDKcQ1PH8yjbpyN7o2sPWg=",
+ 'date' => @time,
+ 'url' => "/bucket/the-key"}, redirect_response('https://bucket.s3-external-3.amazonaws.com/the-key'))
+ EventMachine::MockHttpRequest.register('https://bucket.s3-external-3.amazonaws.com:443/the-key', :put, {
+ "Authorization"=>"AWS abc:lZMKxGDKcQ1PH8yjbpyN7o2sPWg=",
+ 'date' => @time,
+ 'url' => "/bucket/the-key"}, fake_response('Thanks!'))
+
+ @item = Happening::S3::Item.new('bucket', 'the-key', :aws_access_key_id => 'abc', :aws_secret_access_key => '123')
+ run_in_em_loop do
+ @item.put('content')
+
+ EM.add_timer(1) {
+ EM.stop_event_loop
+ assert_equal 1, EventMachine::MockHttpRequest.count('https://bucket.s3.amazonaws.com:443/the-key', :put, {
+ "Authorization"=>"AWS abc:lZMKxGDKcQ1PH8yjbpyN7o2sPWg=",
+ 'date' => @time,
+ 'url' => "/bucket/the-key"})
+
+ assert_equal 1, EventMachine::MockHttpRequest.count('https://bucket.s3-external-3.amazonaws.com:443/the-key', :put, {
+ "Authorization"=>"AWS abc:lZMKxGDKcQ1PH8yjbpyN7o2sPWg=",
+ 'date' => @time,
+ 'url' => "/bucket/the-key"})
+ }
+
+ end
+ end
+
+ should "retry on error" do
+ EventMachine::MockHttpRequest.register('https://bucket.s3.amazonaws.com:443/the-key', :put, {
+ "Authorization"=>"AWS abc:lZMKxGDKcQ1PH8yjbpyN7o2sPWg=",
+ 'date' => @time,
+ 'url' => "/bucket/the-key"}, error_response(400))
+
+ @item = Happening::S3::Item.new('bucket', 'the-key', :aws_access_key_id => 'abc', :aws_secret_access_key => '123')
+ run_in_em_loop do
+ @item.put('content')
+
+ EM.add_timer(1) {
+ EM.stop_event_loop
+ assert_equal 5, EventMachine::MockHttpRequest.count('https://bucket.s3.amazonaws.com:443/the-key', :put, {
+ "Authorization"=>"AWS abc:lZMKxGDKcQ1PH8yjbpyN7o2sPWg=",
+ 'date' => @time,
+ 'url' => "/bucket/the-key"})
+ }
+
+ end
end
+
end
end
View
27 test/test_helper.rb
@@ -57,9 +57,36 @@ def fake_response(data)
Connection: close
#{data}
+HEREDOC
+end
+
+# amazon tells us to upload to another location, e.g. happening-benchmark.s3-external-3.amazonaws.com instead of happening-benchmark.s3.amazonaws.com
+def redirect_response(location)
+ <<-HEREDOC
+HTTP/1.0 301 Moved Permanently
+Date: Mon, 16 Nov 2009 20:39:15 GMT
+Expires: -1
+Cache-Control: private, max-age=0
+Content-Type: text/html; charset=ISO-8859-1
+Via: 1.0 .:80 (squid)
+Connection: close
+Location: #{location}
+
+<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>TemporaryRedirect</Code><Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message><RequestId>137D5486D66095AE</RequestId><Bucket>happening-benchmark</Bucket><HostId>Nyk+Zq9GbtxcspdbKDWyGhsZhyUZquZP55tteYef4QVodsn73HUUad0xrIeD09lF</HostId><Endpoint>#{location}</Endpoint></Error>
HEREDOC
end
+def error_response(error_code)
+ <<-HEREDOC
+HTTP/1.0 #{error_code} OK
+Date: Mon, 16 Nov 2009 20:39:15 GMT
+Content-Type: text/html; charset=ISO-8859-1
+Connection: close
+
+<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>TemporaryRedirect</Code><Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message><RequestId>137D5486D66095AE</RequestId><Bucket>happening-benchmark</Bucket><HostId>Nyk+Zq9GbtxcspdbKDWyGhsZhyUZquZP55tteYef4QVodsn73HUUad0xrIeD09lF</HostId><Endpoint>https://s3.amazonaws.com</Endpoint></Error>
+HEREDOC
+end
+
module EventMachine
class MockHttpRequest
@@pass_through_requests = false
Please sign in to comment.
Something went wrong with that request. Please try again.