Allow to send configuration options to fetcher

config/config.exs

@@ -32,7 +32,7 @@ use Mix.Config
 config :crawly, Crawly.Worker, client: HTTPoison
 
 config :crawly,
-  fetcher: Crawly.Fetchers.HTTPoisonFetcher,
+  fetcher: {Crawly.Fetchers.HTTPoisonFetcher, []},
 
   # User agents which are going to be used with requests
   user_agents: [

lib/crawly/fetchers/fetcher.ex

@@ -2,12 +2,13 @@ defmodule Crawly.Fetchers.Fetcher do
   @moduledoc """
   A behavior module for defining Crawly Fetchers
 
-  A fetcher is expected to implement a fetch callback which should return a
-  Crawly.Response
+  A fetcher is expected to implement a fetch callback which should take
+  Crawly.Request, HTTP client options and return Crawly.Response.
   """
 
-  @callback fetch(request) :: {:ok, response} | {:error, reason}
+  @callback fetch(request, options) :: {:ok, response} | {:error, reason}
             when request: Crawly.Request.t(),
                  response: Crawly.Response.t(),
+                 options: map(),
                  reason: term()
 end

lib/crawly/fetchers/httpoison_fetcher.ex

@@ -6,7 +6,7 @@ defmodule Crawly.Fetchers.HTTPoisonFetcher do
 
   require Logger
 
-  def fetch(request) do
+  def fetch(request, _client_options) do
     HTTPoison.get(request.url, request.headers, request.options)
   end
 end

lib/crawly/worker.ex

@@ -71,14 +71,15 @@ defmodule Crawly.Worker do
              result: {:ok, response, spider_name} | {:error, term()}
   defp get_response({request, spider_name}) do
     # check if spider-level fetcher is set. Overrides the globally configured fetcher.
-    # if not set, log warning for explicit config preferred, get the globally-configured fetcher. Defaults to FwtchWithHTTPoison
-    fetcher = Application.get_env(
+    # if not set, log warning for explicit config preferred,
+    # get the globally-configured fetcher. Defaults to HTTPoisonFetcher
+    {fetcher, options} = Application.get_env(
       :crawly,
       :fetcher,
-      Crawly.Fetchers.HTTPoisonFetcher
+      {Crawly.Fetchers.HTTPoisonFetcher, []}
     )
 
-    case fetcher.fetch(request) do
+    case fetcher.fetch(request, options) do
       {:ok, response} ->
         {:ok, {response, spider_name}}