Merge abee884 into b5ba74c

config/config.exs

@@ -32,6 +32,8 @@ use Mix.Config
 config :crawly, Crawly.Worker, client: HTTPoison
 
 config :crawly,
+  fetcher: {Crawly.Fetchers.HTTPoisonFetcher, []},
+
   # User agents which are going to be used with requests
   user_agents: [
     "Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0",

documentation/configuration.md

@@ -125,3 +125,10 @@ It's possible to set proxy using the proxy value of Crawly config, for example:
 config :crawly,
     proxy: "<proxy_host>:<proxy_port>",
 ```
+
+### fetcher :: atom()
+
+default: Crawly.Fetchers.HTTPoisonFetcher 
+
+Allows to specify a custom HTTP client which will be performing request to the crawl
+target.

lib/crawly/fetchers/fetcher.ex

@@ -0,0 +1,14 @@
+defmodule Crawly.Fetchers.Fetcher do
+  @moduledoc """
+  A behavior module for defining Crawly Fetchers
+
+  A fetcher is expected to implement a fetch callback which should take
+  Crawly.Request, HTTP client options and return Crawly.Response.
+  """
+
+  @callback fetch(request, options) :: {:ok, response} | {:error, reason}
+            when request: Crawly.Request.t(),
+                 response: Crawly.Response.t(),
+                 options: map(),
+                 reason: term()
+end

lib/crawly/fetchers/httpoison_fetcher.ex

@@ -0,0 +1,12 @@
+defmodule Crawly.Fetchers.HTTPoisonFetcher do
+  @moduledoc """
+  Implements Crawly.Fetchers.Fetcher behavior based on HTTPoison HTTP client
+  """
+  @behaviour Crawly.Fetchers.Fetcher
+
+  require Logger
+
+  def fetch(request, _client_options) do
+    HTTPoison.get(request.url, request.headers, request.options)
+  end
+end

lib/crawly/request.ex

@@ -7,13 +7,15 @@ defmodule Crawly.Request do
   defstruct url: nil, headers: [], prev_response: nil, options: []
 
   @type header() :: {key(), value()}
+  @type url() :: binary()
+
   @typep key :: binary()
   @typep value :: binary()
 
   @type option :: {atom(), binary()}
 
   @type t :: %__MODULE__{
-    url: binary(),
+    url: url(),
     headers: [header()],
     prev_response: %{},
     options: [option()]

lib/crawly/response.ex

@@ -0,0 +1,19 @@
+defmodule Crawly.Response do
+  @moduledoc """
+  Define Crawly response structure
+  """
+
+  defstruct body: nil,
+            headers: [],
+            request: nil,
+            request_url: nil,
+            status_code: nil
+
+  @type t :: %__MODULE__{
+               body: term(),
+               headers: list(),
+               request: Crawly.Request.t(),
+               request_url: Crawly.Request.url(),
+               status_code: integer()
+             }
+end

lib/crawly/worker.ex

@@ -45,14 +45,18 @@ defmodule Crawly.Worker do
               # TODO: Add retry logic
               Logger.error(
                 fn ->
-                  "Crawly worker could not process the request to #{inspect(request.url)}
+                  "Crawly worker could not process the request to #{
+                    inspect(request.url)
+                  }
                   reason: #{inspect(reason)}"
-                end)
+                end
+              )
+
               @default_backoff
+
             {:ok, _result} ->
               @default_backoff
           end
-
       end
 
     Process.send_after(self(), :work, new_backoff)
@@ -66,7 +70,16 @@ defmodule Crawly.Worker do
              response: HTTPoison.Response.t(),
              result: {:ok, response, spider_name} | {:error, term()}
   defp get_response({request, spider_name}) do
-    case HTTPoison.get(request.url, request.headers, request.options) do
+    # check if spider-level fetcher is set. Overrides the globally configured fetcher.
+    # if not set, log warning for explicit config preferred,
+    # get the globally-configured fetcher. Defaults to HTTPoisonFetcher
+    {fetcher, options} = Application.get_env(
+      :crawly,
+      :fetcher,
+      {Crawly.Fetchers.HTTPoisonFetcher, []}
+    )
+
+    case fetcher.fetch(request, options) do
       {:ok, response} ->
         {:ok, {response, spider_name}}