Permalink
Browse files

Added excon and made errors consistent.

  • Loading branch information...
2 parents 9d5dcf6 + eae49d3 commit 6331f6da1066726c76ac05bbddc917d3b6b8669f @treeder treeder committed Jul 10, 2012
View
@@ -11,6 +11,7 @@ GEM
excon (0.14.3)
ffi (1.0.11)
mime-types (1.19)
+ minitest (3.2.0)
net-http-persistent (2.7)
quicky (0.0.1)
rake (0.9.2.2)
@@ -27,6 +28,8 @@ PLATFORMS
DEPENDENCIES
excon
+ minitest
+ net-http-persistent
quicky
rake
rest!
View
@@ -71,10 +71,13 @@ The response object you get back will always be consistent and will have the fol
response.body
-Errors
+Exceptions
======
-If it didn't get a response, you will get a Rest::ClientError
+If it didn't get a response for whatever reason, you will get a Rest::ClientError
-If status code is 40X or 50X, you will get a Rest::HttpError.
+If status code is 40X or 50X, it will raise an exception with the following methods.
+
+ err.code
+ err.response (which has body: err.response.body)
View
@@ -6,6 +6,7 @@ Rake::TestTask.new(:test) do |test|
test.libs << 'lib' << 'test'
test.pattern = 'test/**/test_*.rb'
test.verbose = true
+ p ENV['gem']
end
task :default => :test
View
@@ -28,6 +28,8 @@ def initialize(options={})
choose_best_gem()
end
+ p @gem
+
if @gem == :excon
require_relative 'wrappers/excon_wrapper'
@wrapper = Rest::Wrappers::ExconWrapper.new(self)
@@ -51,85 +53,120 @@ def choose_best_gem
begin
raise LoadError
require 'typhoeus'
- @gem = :client
+ @gem = :typhoeus
rescue LoadError => ex
begin
# try net-http-persistent
require 'net/http/persistent'
@gem = :net_http_persistent
rescue LoadError => ex
- require 'rest_client'
- @gem = :rest_client
end
end
+ if @gem.nil?
+ require 'rest_client'
+ @gem = :rest_client
+ end
end
def get(url, req_hash={})
res = nil
- perform_op do
+ res = perform_op(:get, req_hash) do
res = @wrapper.get(url, req_hash)
end
return res
end
# This will attempt to perform the operation with an exponential backoff on 503 errors.
# Amazon services throw 503
- def perform_op(&blk)
- max_retries = @options[:max_retries] || 5
+ # todo: just make perform_op a method and have it call the wrapper. The block is a waste now.
+ def perform_op(method, req_hash, options={}, &blk)
+ set_defaults(options)
+ max_retries = options[:max_retries] || 5
+ max_follows = options[:max_follows] || 10
+ if options[:follow_count] && options[:follow_count] >= max_follows
+ raise Rest::RestError "Too many follows. #{options[:follow_count]}"
+ end
current_retry = 0
+ current_follow = 0
success = false
res = nil
- while current_retry < max_retries do
- res = yield blk
- #p res
- #p res.code
- if res.code == 503
- pow = (4 ** (current_retry)) * 100 # milliseconds
- #puts 'pow=' + pow.to_s
- s = Random.rand * pow
- #puts 's=' + s.to_s
- sleep_secs = 1.0 * s / 1000.0
- puts 'sleep for ' + sleep_secs.to_s
- current_retry += 1
- @logger.debug "503 Received. Retrying #{current_retry} out of #{max_retries} max in #{sleep_secs} seconds."
- sleep sleep_secs
- else
+ while current_retry < max_retries && current_follow < max_follows do
+ begin
+ res = yield blk
+ res.tries = current_retry + 1
+ if res.code >= 300 && res.code < 400
+ # try new location
+ #p res.headers
+ loc = res.headers["location"]
+ @logger.debug "#{res.code} Received. Trying new location: #{loc}"
+ if loc.nil?
+ raise InvalidResponseError.new("No location header received with #{res.code} status code!")
+ end
+ # options.merge({:max_follows=>options[:max_follows-1]}
+ options[:follow_count] ||= 0
+ options[:follow_count] += 1
+ res = perform_op(method, req_hash, options) do
+ res = @wrapper.send(method, loc, req_hash)
+ end
+ #puts 'X: ' + res.inspect
+ return res
+ end
+ # If it's here, then it's all good
break
+ rescue Rest::HttpError => ex
+ if ex.code == 503
+ pow = (4 ** (current_retry)) * 100 # milliseconds
+ #puts 'pow=' + pow.to_s
+ s = Random.rand * pow
+ #puts 's=' + s.to_s
+ sleep_secs = 1.0 * s / 1000.0
+ #puts 'sleep for ' + sleep_secs.to_s
+ current_retry += 1
+ @logger.debug "#{ex.code} Received. Retrying #{current_retry} out of #{max_retries} max in #{sleep_secs} seconds."
+ sleep sleep_secs
+ else
+ raise ex
+ end
end
end
res
end
+ def set_defaults(options)
+ options[:max_retries] ||= (@options[:max_retries] || 5)
+ options[:max_follows] ||= (@options[:max_follows] || 10)
+ end
+
# req_hash options:
# - :body => post body
#
def post(url, req_hash={})
res = nil
- perform_op do
+ res = perform_op(:post, req_hash) do
res = @wrapper.post(url, req_hash)
end
return res
end
def put(url, req_hash={})
res = nil
- perform_op do
+ res = perform_op(:put, req_hash) do
res = @wrapper.put(url, req_hash)
end
return res
end
def delete(url, req_hash={})
res = nil
- perform_op do
+ res = perform_op(:delete, req_hash) do
res = @wrapper.delete(url, req_hash)
end
return res
end
def post_file(url, req_hash={})
res = nil
- perform_op do
+ res = perform_op(:post_file, req_hash) do
res = @wrapper.post_file(url, req_hash)
end
return res
View
@@ -7,7 +7,17 @@ class RestError < StandardError
end
class HttpError < RestError
+ def initialize(response)
+ super("#{response.code} Error")
+ @response = response
+ end
+ def response
+ @response
+ end
+ def code
+ response.code
+ end
end
# If it didn't even get a response, it will be a ClientError
@@ -21,5 +31,9 @@ def initialize(msg=nil)
super(msg)
end
end
+
+ class InvalidResponseError < RestError
+
+ end
end
@@ -1,29 +1,53 @@
-class BaseWrapper
-
- def post_file(url, req_hash={})
- response = nil
- begin
- if req_hash[:body]
- req_hash = req_hash.merge(req_hash[:body])
- req_hash.delete(:body)
- end
+module Rest
+ class BaseWrapper
+
+ def post_file(url, req_hash={})
+ response = nil
+ begin
+ if req_hash[:body]
+ req_hash = req_hash.merge(req_hash[:body])
+ req_hash.delete(:body)
+ end
+
+ headers = {}
+ if req_hash[:headers]
+ headers = req_hash[:headers]
+ req_hash.delete(:headers)
+ end
- headers = {}
- if req_hash[:headers]
- headers = req_hash[:headers]
- req_hash.delete(:headers)
+ r2 = RestClient.post(url, req_hash, headers)
+ response = Rest::Wrappers::RestClientResponseWrapper.new(r2)
+ rescue RestClient::Exception => ex
+ raise Rest::Wrappers::RestClientExceptionWrapper.new(ex)
end
+ response
+ end
+
+ # if body is a hash, it will convert it to json
+ def to_json_parts(h)
+ h[:body] = h[:body].to_json if h[:body] && h[:body].is_a?(Hash)
+ end
+
+ # if wrapper has a close/shutdown, override this
+ def close
- r2 = RestClient.post(url, req_hash, headers)
- response = Rest::Wrappers::RestClientResponseWrapper.new(r2)
- rescue RestClient::Exception => ex
- raise Rest::Wrappers::RestClientExceptionWrapper.new(ex)
end
- response
end
- # if wrapper has a close/shutdown, override this
- def close
+ class BaseResponseWrapper
+ attr_accessor :tries
+
+ # Provide a headers_orig method in your wrapper to allow this to work
+ def headers
+ new_h = {}
+ headers_orig.each_pair do |k,v|
+ if v.is_a?(Array) && v.size == 1
+ v = v[0]
+ end
+ new_h[k.downcase] = v
+ end
+ new_h
+ end
end
end
@@ -10,7 +10,7 @@ def initialize(ex)
end
end
- class ExconResponseWrapper
+ class ExconResponseWrapper < BaseResponseWrapper
def initialize(response)
@response = response
end
@@ -23,6 +23,10 @@ def body
@response.body
end
+ def headers_orig
+ @response.headers
+ end
+
end
class ExconWrapper < BaseWrapper
@@ -49,10 +53,7 @@ def get(url, req_hash={})
response = excon_request(url, req_hash)
rescue RestClient::Exception => ex
#p ex
- if ex.http_code == 404
- return RestClientResponseWrapper.new(ex.response)
- end
- raise RestClientExceptionWrapper.new(ex)
+ raise ExconExceptionWrapper.new(ex)
end
response
end
@@ -61,16 +62,21 @@ def excon_request(url, req_hash)
conn = Excon.new(url)
r2 = conn.request(req_hash)
response = ExconResponseWrapper.new(r2)
+ if response.code >= 400
+ raise HttpError.new(response)
+ end
+ response
end
def post(url, req_hash={})
response = nil
begin
req_hash[:method] = :post
req_hash[:url] = url
+ to_json_parts(req_hash)
response = excon_request(url, req_hash)
rescue RestClient::Exception => ex
- raise RestClientExceptionWrapper.new(ex)
+ raise HttpError.new(ex)
end
response
end
Oops, something went wrong.

0 comments on commit 6331f6d

Please sign in to comment.