Merge 3a5517d into ac9f864

wcc-contentful/README.md

@@ -312,8 +312,14 @@ WCC::Contentful.configure do |config|
   # incompatible with config.content_delivery
   # config.store = MyCustomStore.new
 
-  # Implement some adapter like this to use another HTTP client
-  config.http_adapter = MyFaradayAdapter.new
+  # Use a custom Faraday connection
+  config.connection = Faraday.new do |builder|
+    f.request :retry
+    f.request MyFaradayRequestAdapter.new
+    ...
+  end
+  # OR implement some adapter like this to use another HTTP client
+  config.connection = MyNetHttpAdapter.new
 
   config.update_schema_file = :never
 end

wcc-contentful/app/jobs/wcc/contentful/webhook_enable_job.rb

@@ -54,7 +54,7 @@ def default_configuration
         space: config.space,
         environment: config.environment,
         default_locale: config.default_locale,
-        adapter: config.http_adapter,
+        connection: config.connection,
         webhook_username: config.webhook_username,
         webhook_password: config.webhook_password
       }

wcc-contentful/lib/wcc/contentful/configuration.rb

@@ -17,7 +17,8 @@ class WCC::Contentful::Configuration
     content_delivery_params
     middleware
     store
-    http_adapter
+    connection
+    connection_options
     update_schema_file
     schema_file
   ].freeze
@@ -129,12 +130,16 @@ def store=(value)
 
   attr_reader :store
 
-  # Sets the adapter which is used to make HTTP requests.
-  # If left unset, the gem attempts to load either 'http' or 'typhoeus'.
-  # You can pass your own adapter which responds to 'call', or even a lambda
-  # that accepts the following parameters:
-  #  ->(url, query, headers = {}, proxy = {}) { ... }
-  attr_accessor :http_adapter
+  # Sets the connection which is used to make HTTP requests.
+  # If left unset, the gem attempts to load 'faraday', 'http' or 'typhoeus'.
+  # You can pass your own adapter which responds to 'get' and 'post', and returns
+  # a response that quacks like Faraday.
+  attr_accessor :connection
+
+  # Sets the connection options which are given to the client.  This can include
+  # an alternative Cdn API URL, timeouts, etc.
+  # See WCC::Contentful::SimpleClient constructor for details.
+  attr_accessor :connection_options
 
   # Indicates whether to update the contentful-schema.json file for building models.
   # The schema can also be updated with `rake wcc_contentful:download_schema`
@@ -174,6 +179,11 @@ def schema_file
   def initialize
     @access_token = ''
     @app_url = ENV['APP_URL']
+    @connection_options = {
+      api_url: 'https://cdn.contentful.com/',
+      preview_api_url: 'https://preview.contentful.com/',
+      management_api_url: 'https://api.contentful.com'
+    }
     @management_token = ''
     @preview_token = ''
     @space = ''

wcc-contentful/lib/wcc/contentful/services.rb

@@ -58,10 +58,11 @@ def client
       @client ||=
         ensure_configured do |config|
           WCC::Contentful::SimpleClient::Cdn.new(
+            **config.connection_options,
             access_token: config.access_token,
             space: config.space,
             default_locale: config.default_locale,
-            adapter: config.http_adapter,
+            connection: config.connection,
             environment: config.environment
           )
         end
@@ -76,10 +77,11 @@ def preview_client
         ensure_configured do |config|
           if config.preview_token.present?
             WCC::Contentful::SimpleClient::Preview.new(
+              **config.connection_options,
               preview_token: config.preview_token,
               space: config.space,
               default_locale: config.default_locale,
-              adapter: config.http_adapter,
+              connection: config.connection,
               environment: config.environment
             )
           end
@@ -95,10 +97,11 @@ def management_client
         ensure_configured do |config|
           if config.management_token.present?
             WCC::Contentful::SimpleClient::Management.new(
+              **config.connection_options,
               management_token: config.management_token,
               space: config.space,
               default_locale: config.default_locale,
-              adapter: config.http_adapter,
+              connection: config.connection,
               environment: config.environment
             )
           end

wcc-contentful/lib/wcc/contentful/simple_client.rb

@@ -16,8 +16,8 @@ module WCC::Contentful
   # `get`.  This method returns a WCC::Contentful::SimpleClient::Response
   # that handles paging automatically.
   #
-  # The SimpleClient by default uses 'http' to perform the gets, but any HTTP
-  # client can be injected by passing a proc as the `adapter:` option.
+  # The SimpleClient by default uses 'faraday' to perform the gets, but any HTTP
+  # client adapter be injected by passing the `connection:` option.
   #
   # @api Client
   class SimpleClient
@@ -30,21 +30,24 @@ class SimpleClient
     # @param [String] space The Space ID to access
     # @param [String] access_token A Contentful Access Token to be sent in the Authorization header
     # @param [Hash] options The remaining optional parameters, defined below
-    # @option options [Symbol, Object] adapter The Adapter to use to make requests.
+    # @option options [Symbol, Object] connection The Faraday connection to use to make requests.
     #   Auto-discovered based on what gems are installed if this is not provided.
     # @option options [String] default_locale The locale query param to set by default.
     # @option options [String] environment The contentful environment to access. Defaults to 'master'.
     # @option options [Boolean] no_follow_redirects If true, do not follow 300 level redirects.
+    # @option options [Number] rate_limit_wait_timeout The maximum time to block the thread waiting
+    #   on a rate limit response.  By default will wait for one 429 and then fail on the second 429.
     def initialize(api_url:, space:, access_token:, **options)
       @api_url = URI.join(api_url, '/spaces/', space + '/')
       @space = space
       @access_token = access_token
 
-      @adapter = SimpleClient.load_adapter(options[:adapter])
+      @adapter = SimpleClient.load_adapter(options[:connection])
 
       @options = options
       @query_defaults = {}
       @query_defaults[:locale] = @options[:default_locale] if @options[:default_locale]
+      @rate_limit_wait_timeout = @options[:rate_limit_wait_timeout] || 1.5
 
       return unless options[:environment].present?
 
@@ -63,6 +66,7 @@ def get(path, query = {})
     end
 
     ADAPTERS = {
+      faraday: ['faraday', '~> 0.9'],
       http: ['http', '> 1.0', '< 3.0'],
       typhoeus: ['typhoeus', '~> 1.0']
     }.freeze
@@ -80,37 +84,58 @@ def self.load_adapter(adapter)
         end
         raise ArgumentError, 'Unable to load adapter!  Please install one of '\
           "#{ADAPTERS.values.map(&:join).join(',')}"
+      when :faraday
+        require 'faraday'
+        ::Faraday.new do |faraday|
+          faraday.response :logger, (Rails.logger if defined?(Rails)), { headers: false, bodies: false }
+          faraday.adapter :net_http
+        end
       when :http
         require_relative 'simple_client/http_adapter'
         HttpAdapter.new
       when :typhoeus
         require_relative 'simple_client/typhoeus_adapter'
         TyphoeusAdapter.new
       else
-        unless adapter.respond_to?(:call)
+        unless adapter.respond_to?(:get)
           raise ArgumentError, "Adapter #{adapter} is not invokeable!  Please "\
-            "pass a proc or use one of #{ADAPTERS.keys}"
+            "pass use one of #{ADAPTERS.keys} or create a Faraday-compatible adapter"
         end
         adapter
       end
     end
 
     private
 
-    def get_http(url, query, headers = {}, proxy = {})
+    def get_http(url, query, headers = {})
       headers = {
         Authorization: "Bearer #{@access_token}"
       }.merge(headers || {})
 
       q = @query_defaults.dup
       q = q.merge(query) if query
 
-      resp = @adapter.call(url, q, headers, proxy)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      loop do
+        resp = @adapter.get(url, q, headers)
+
+        if [301, 302, 307].include?(resp.status) && !@options[:no_follow_redirects]
+          url = resp.headers['Location']
+          next
+        end
+
+        if resp.status == 429 &&
+            reset = resp.headers['X-Contentful-RateLimit-Reset'].presence
+          reset = reset.to_f
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          if (now - start) + reset < @rate_limit_wait_timeout
+            sleep(reset)
+            next
+          end
+        end
 
-      if [301, 302, 307].include?(resp.code) && !@options[:no_follow_redirects]
-        resp = get_http(resp.headers['location'], nil, headers, proxy)
+        return resp
       end
-      resp
     end
 
     # The CDN SimpleClient accesses 'https://cdn.contentful.com' to get raw
@@ -186,10 +211,10 @@ def sync(sync_token: nil, **query)
     class Preview < Cdn
       def initialize(space:, preview_token:, **options)
         super(
-          api_url: options[:api_url] || 'https://preview.contentful.com/',
+          **options,
+          api_url: options[:preview_api_url] || 'https://preview.contentful.com/',
           space: space,
-          access_token: preview_token,
-          **options
+          access_token: preview_token
         )
       end
 

wcc-contentful/lib/wcc/contentful/simple_client/http_adapter.rb

@@ -3,22 +3,36 @@
 gem 'http'
 require 'http'
 
-class HttpAdapter
-  def call(url, query, headers = {}, proxy = {})
-    if proxy[:host]
-      HTTP[headers].via(proxy[:host], proxy[:port], proxy[:username], proxy[:password])
-        .get(url, params: query)
-    else
-      HTTP[headers].get(url, params: query)
-    end
+class WCC::Contentful::SimpleClient::HttpAdapter
+  def get(url, params = {}, headers = {})
+    req = OpenStruct.new(params: params, headers: headers)
+
+    yield req if block_given?
+
+    Response.new(
+      HTTP[req.headers].get(url, params: req.params)
+    )
   end
 
   def post(url, body, headers = {}, proxy = {})
-    if proxy[:host]
-      HTTP[headers].via(proxy[:host], proxy[:port], proxy[:username], proxy[:password])
-        .post(url, json: body)
-    else
-      HTTP[headers].post(url, json: body)
-    end
+    Response.new(
+      if proxy[:host]
+        HTTP[headers].via(proxy[:host], proxy[:port], proxy[:username], proxy[:password])
+          .post(url, json: body)
+      else
+        HTTP[headers].post(url, json: body)
+      end
+    )
   end
+
+  Response =
+    Struct.new(:raw) do
+      extend Forwardable
+
+      def_delegators :raw, :body, :to_s, :status, :headers
+
+      def status
+        raw.code
+      end
+    end
 end

wcc-contentful/lib/wcc/contentful/simple_client/management.rb

@@ -4,10 +4,10 @@
 class WCC::Contentful::SimpleClient::Management < WCC::Contentful::SimpleClient
   def initialize(space:, management_token:, **options)
     super(
-      api_url: options[:api_url] || 'https://api.contentful.com',
+      **options,
+      api_url: options[:management_api_url] || 'https://api.contentful.com',
       space: space,
       access_token: management_token,
-      **options
     )
 
     @post_adapter = @adapter if @adapter.respond_to?(:post)
@@ -75,16 +75,19 @@ def post(path, body)
 
   private
 
-  def post_http(url, body, headers = {}, proxy = {})
+  def post_http(url, body, headers = {})
     headers = {
       Authorization: "Bearer #{@access_token}",
       'Content-Type' => 'application/vnd.contentful.management.v1+json'
     }.merge(headers || {})
 
-    resp = @post_adapter.post(url, body, headers, proxy)
+    body = body.to_json unless body.is_a? String
+    resp = @post_adapter.post(url, body, headers)
 
-    if [301, 302, 307].include?(resp.code) && !@options[:no_follow_redirects]
-      resp = get_http(resp.headers['location'], nil, headers, proxy)
+    if [301, 302, 307].include?(resp.status) && !@options[:no_follow_redirects]
+      resp = get_http(resp.headers['location'], nil, headers)
+    elsif resp.status == 308 && !@options[:no_follow_redirects]
+      resp = post_http(resp.headers['location'], body, headers)
     end
     resp
   end

wcc-contentful/lib/wcc/contentful/simple_client/response.rb

@@ -6,7 +6,8 @@ class Response
     attr_reader :client
     attr_reader :request
 
-    delegate :code, to: :raw_response
+    delegate :status, to: :raw_response
+    alias_method :code, :status
     delegate :headers, to: :raw_response
 
     def body
@@ -62,9 +63,9 @@ def initialize(client, request, raw_response)
     end
 
     def assert_ok!
-      return self if code >= 200 && code < 300
+      return self if status >= 200 && status < 300
 
-      raise ApiError[code], self
+      raise ApiError[status], self
     end
 
     def each_page(&block)
@@ -186,6 +187,8 @@ def self.[](code)
         NotFoundError
       when 401
         UnauthorizedError
+      when 429
+        RateLimitError
       else
         ApiError
       end
@@ -202,4 +205,7 @@ class NotFoundError < ApiError
 
   class UnauthorizedError < ApiError
   end
+
+  class RateLimitError < ApiError
+  end
 end

wcc-contentful/lib/wcc/contentful/simple_client/typhoeus_adapter.rb

@@ -3,23 +3,23 @@
 gem 'typhoeus'
 require 'typhoeus'
 
-class TyphoeusAdapter
-  def call(url, query, headers = {}, proxy = {})
-    raise NotImplementedError, 'Proxying Not Yet Implemented' if proxy[:host]
-
-    TyphoeusAdapter::Response.new(
+class WCC::Contentful::SimpleClient::TyphoeusAdapter
+  def get(url, params = {}, headers = {})
+    req = OpenStruct.new(params: params, headers: headers)
+    yield req if block_given?
+    Response.new(
       Typhoeus.get(
         url,
-        params: query,
-        headers: headers
+        params: req.params,
+        headers: req.headers
       )
     )
   end
 
   def post(url, body, headers = {}, proxy = {})
     raise NotImplementedError, 'Proxying Not Yet Implemented' if proxy[:host]
 
-    TyphoeusAdapter::Response.new(
+    Response.new(
       Typhoeus.post(
         url,
         body: body.to_json,