Added excon and made errors consistent.

Gemfile.lock

@@ -11,6 +11,7 @@ GEM
     excon (0.14.3)
     ffi (1.0.11)
     mime-types (1.19)
+    minitest (3.2.0)
     net-http-persistent (2.7)
     quicky (0.0.1)
     rake (0.9.2.2)
@@ -27,6 +28,8 @@ PLATFORMS
 
 DEPENDENCIES
   excon
+  minitest
+  net-http-persistent
   quicky
   rake
   rest!

README.md

@@ -71,10 +71,13 @@ The response object you get back will always be consistent and will have the fol
     response.body
 
 
-Errors
+Exceptions
 ======
 
-If it didn't get a response, you will get a Rest::ClientError
+If it didn't get a response for whatever reason, you will get a Rest::ClientError
 
-If status code is 40X or 50X, you will get a Rest::HttpError.
+If status code is 40X or 50X, it will raise an exception with the following methods.
+
+    err.code
+    err.response (which has body: err.response.body)
 

Rakefile

@@ -6,6 +6,7 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
+  p ENV['gem']
 end
 
 task :default => :test

lib/rest/client.rb

@@ -28,6 +28,8 @@ def initialize(options={})
         choose_best_gem()
       end
 
+      p @gem
+
       if @gem == :excon
         require_relative 'wrappers/excon_wrapper'
         @wrapper = Rest::Wrappers::ExconWrapper.new(self)
@@ -51,85 +53,120 @@ def choose_best_gem
       begin
         raise LoadError
         require 'typhoeus'
-        @gem = :client
+        @gem = :typhoeus
       rescue LoadError => ex
         begin
           # try net-http-persistent
           require 'net/http/persistent'
           @gem = :net_http_persistent
         rescue LoadError => ex
-         require 'rest_client'
-          @gem = :rest_client
         end
       end
+      if @gem.nil?
+        require 'rest_client'
+        @gem = :rest_client
+      end
     end
 
     def get(url, req_hash={})
       res = nil
-      perform_op do
+      res = perform_op(:get, req_hash) do
         res = @wrapper.get(url, req_hash)
       end
       return res
     end
 
     # This will attempt to perform the operation with an exponential backoff on 503 errors.
     # Amazon services throw 503
-    def perform_op(&blk)
-      max_retries = @options[:max_retries] || 5
+    # todo: just make perform_op a method and have it call the wrapper. The block is a waste now.
+    def perform_op(method, req_hash, options={}, &blk)
+      set_defaults(options)
+      max_retries = options[:max_retries] || 5
+      max_follows = options[:max_follows] || 10
+      if options[:follow_count] && options[:follow_count] >= max_follows
+        raise Rest::RestError "Too many follows. #{options[:follow_count]}"
+      end
       current_retry = 0
+      current_follow = 0
       success = false
       res = nil
-      while current_retry < max_retries do
-        res = yield blk
-        #p res
-        #p res.code
-        if res.code == 503
-          pow = (4 ** (current_retry)) * 100 # milliseconds
-                                             #puts 'pow=' + pow.to_s
-          s = Random.rand * pow
-                                             #puts 's=' + s.to_s
-          sleep_secs = 1.0 * s / 1000.0
-                                             puts 'sleep for ' + sleep_secs.to_s
-          current_retry += 1
-          @logger.debug "503 Received. Retrying #{current_retry} out of #{max_retries} max in #{sleep_secs} seconds."
-          sleep sleep_secs
-        else
+      while current_retry < max_retries && current_follow < max_follows do
+        begin
+          res = yield blk
+          res.tries = current_retry + 1
+          if res.code >= 300 && res.code < 400
+            # try new location
+            #p res.headers
+            loc = res.headers["location"]
+            @logger.debug "#{res.code} Received. Trying new location: #{loc}"
+            if loc.nil?
+              raise InvalidResponseError.new("No location header received with #{res.code} status code!")
+            end
+            # options.merge({:max_follows=>options[:max_follows-1]}
+            options[:follow_count] ||= 0
+            options[:follow_count] += 1
+            res = perform_op(method, req_hash, options) do
+              res = @wrapper.send(method, loc, req_hash)
+            end
+            #puts 'X: ' + res.inspect
+            return res
+          end
+          # If it's here, then it's all good
           break
+        rescue Rest::HttpError => ex
+          if ex.code == 503
+            pow = (4 ** (current_retry)) * 100 # milliseconds
+                                               #puts 'pow=' + pow.to_s
+            s = Random.rand * pow
+                                               #puts 's=' + s.to_s
+            sleep_secs = 1.0 * s / 1000.0
+                                               #puts 'sleep for ' + sleep_secs.to_s
+            current_retry += 1
+            @logger.debug "#{ex.code} Received. Retrying #{current_retry} out of #{max_retries} max in #{sleep_secs} seconds."
+            sleep sleep_secs
+          else
+            raise ex
+          end
         end
       end
       res
     end
 
+    def set_defaults(options)
+      options[:max_retries] ||= (@options[:max_retries] || 5)
+      options[:max_follows] ||= (@options[:max_follows] || 10)
+    end
+
     # req_hash options:
     # - :body => post body
     #
     def post(url, req_hash={})
       res = nil
-      perform_op do
+      res = perform_op(:post, req_hash) do
         res = @wrapper.post(url, req_hash)
       end
       return res
     end
 
     def put(url, req_hash={})
       res = nil
-      perform_op do
+      res = perform_op(:put, req_hash) do
         res = @wrapper.put(url, req_hash)
       end
       return res
     end
 
     def delete(url, req_hash={})
       res = nil
-      perform_op do
+      res = perform_op(:delete, req_hash) do
         res = @wrapper.delete(url, req_hash)
       end
       return res
     end
 
     def post_file(url, req_hash={})
       res = nil
-      perform_op do
+      res = perform_op(:post_file, req_hash) do
         res = @wrapper.post_file(url, req_hash)
       end
       return res

lib/rest/errors.rb

@@ -7,7 +7,17 @@ class RestError < StandardError
   end
 
   class HttpError < RestError
+    def initialize(response)
+      super("#{response.code} Error")
+      @response = response
+    end
 
+    def response
+      @response
+    end
+    def code
+      response.code
+    end
   end
 
   # If it didn't even get a response, it will be a ClientError
@@ -21,5 +31,9 @@ def initialize(msg=nil)
       super(msg)
     end
   end
+
+  class InvalidResponseError < RestError
+
+  end
 end
 

lib/rest/wrappers/base_wrapper.rb

@@ -1,29 +1,53 @@
-class BaseWrapper
-
-  def post_file(url, req_hash={})
-    response = nil
-    begin
-      if req_hash[:body]
-        req_hash = req_hash.merge(req_hash[:body])
-        req_hash.delete(:body)
-      end
+module Rest
+  class BaseWrapper
+
+    def post_file(url, req_hash={})
+      response = nil
+      begin
+        if req_hash[:body]
+          req_hash = req_hash.merge(req_hash[:body])
+          req_hash.delete(:body)
+        end
+
+        headers = {}
+        if req_hash[:headers]
+          headers = req_hash[:headers]
+          req_hash.delete(:headers)
+        end
 
-      headers = {}
-      if req_hash[:headers]
-        headers = req_hash[:headers]
-        req_hash.delete(:headers)
+        r2 = RestClient.post(url, req_hash, headers)
+        response = Rest::Wrappers::RestClientResponseWrapper.new(r2)
+      rescue RestClient::Exception => ex
+        raise Rest::Wrappers::RestClientExceptionWrapper.new(ex)
       end
+      response
+    end
+
+    # if body is a hash, it will convert it to json
+    def to_json_parts(h)
+      h[:body] = h[:body].to_json if h[:body] && h[:body].is_a?(Hash)
+    end
+
+    # if wrapper has a close/shutdown, override this
+    def close
 
-      r2 = RestClient.post(url, req_hash, headers)
-      response = Rest::Wrappers::RestClientResponseWrapper.new(r2)
-    rescue RestClient::Exception => ex
-      raise Rest::Wrappers::RestClientExceptionWrapper.new(ex)
     end
-    response
   end
 
-  # if wrapper has a close/shutdown, override this
-  def close
+  class BaseResponseWrapper
+    attr_accessor :tries
+
+    # Provide a headers_orig method in your wrapper to allow this to work
+    def headers
+      new_h = {}
+      headers_orig.each_pair do |k,v|
+        if v.is_a?(Array) && v.size == 1
+          v = v[0]
+        end
+        new_h[k.downcase] = v
+      end
+      new_h
+    end
 
   end
 end

lib/rest/wrappers/excon_wrapper.rb

@@ -10,7 +10,7 @@ def initialize(ex)
       end
     end
 
-    class ExconResponseWrapper
+    class ExconResponseWrapper < BaseResponseWrapper
       def initialize(response)
         @response = response
       end
@@ -23,6 +23,10 @@ def body
         @response.body
       end
 
+      def headers_orig
+        @response.headers
+      end
+
     end
 
     class ExconWrapper < BaseWrapper
@@ -49,10 +53,7 @@ def get(url, req_hash={})
           response = excon_request(url, req_hash)
         rescue RestClient::Exception => ex
           #p ex
-          if ex.http_code == 404
-            return RestClientResponseWrapper.new(ex.response)
-          end
-          raise RestClientExceptionWrapper.new(ex)
+          raise ExconExceptionWrapper.new(ex)
         end
         response
       end
@@ -61,16 +62,21 @@ def excon_request(url, req_hash)
         conn = Excon.new(url)
         r2 = conn.request(req_hash)
         response = ExconResponseWrapper.new(r2)
+        if response.code >= 400
+          raise HttpError.new(response)
+        end
+        response
       end
 
       def post(url, req_hash={})
         response = nil
         begin
           req_hash[:method] = :post
           req_hash[:url] = url
+          to_json_parts(req_hash)
           response = excon_request(url, req_hash)
         rescue RestClient::Exception => ex
-          raise RestClientExceptionWrapper.new(ex)
+          raise HttpError.new(ex)
         end
         response
       end