PR cleanups

lib/crawler/fetcher/modifier.ex

@@ -1,54 +1,45 @@
 defmodule Crawler.Fetcher.Modifier do
-
-    @moduledoc """
-        modify request options and headers before dispatch 
-    """
+  @moduledoc """
+  Modifies request options and headers before dispatch.
+  """
 
-    defmodule Spec do
-        @moduledoc """
-        Spec for defining a request modifier.
-        """
+  defmodule Spec do
+    @type url    :: String.t
+    @type header :: {String.t, String.t}
+    @type opts   :: map
 
-        @type url  :: String.t
-        @type header :: {String.t, String.t}
-        @type opts :: map
+    @callback headers(opts) :: list(header) | []
+    @callback opts(opts)    :: keyword | []
+  end
 
-        @callback headers(opts) :: list(header) | []
-        @callback opts(opts)   :: keyword | []
+  @behaviour __MODULE__.Spec
 
+  @doc """
+  Allows modifing headers prior to making a crawl request.
+
+  ## Example implementaion
+
+    def headers(opts) do
+      if opts[:url] == "http://modifier" do
+        [{"Referer", "http://fetcher"}]
+      end
+      []
     end
+  """
+  def headers(_opts), do: []
+
+  @doc """
+  Allows passing opts to httpPoison prior to making the crawl request
 
-    @behaviour __MODULE__.Spec
-
-    @doc """
-    Allows modifing headers prior to making the crawl request
-
-    ## Example implementaion
-
-        def headers(opts) do
-            if opts[:url] == "http://modifier" do
-                [{"Referer", "http://fetcher"}]  
-            end
-            []
-        end
-
-    """
-    def headers(_opts), do: []
-
-    @doc """
-    Allows passing opts to httpPoison prior to making the crawl request
-      
-    ## Example implementation
- 
-        def opts(opts) do
-            if opts[:url] == "http://modifier" do
-                # add a new pool to hackney
-                [hackney: [pool: :modifier]] 
-            end
-            []
-        end
-        
-    """
-    def opts(_opts), do: []
-
-end
+  ## Example implementation
+
+    def opts(opts) do
+      if opts[:url] == "http://modifier" do
+        # add a new pool to hackney
+        [hackney: [pool: :modifier]]
+      end
+      []
+    end
+  """
+  def opts(_opts), do: []
+end

lib/crawler/fetcher/requester.ex

@@ -23,13 +23,10 @@ defmodule Crawler.Fetcher.Requester do
   end
 
   defp fetch_headers(opts) do
-    [{"User-Agent", opts[:user_agent]}]
-    ++ opts[:modifier].headers(opts)
+    [{"User-Agent", opts[:user_agent]}] ++ opts[:modifier].headers(opts)
   end
 
   defp fetch_opts(opts) do
-    @fetch_opts ++ [
-      recv_timeout: opts[:timeout]
-    ] ++ opts[:modifier].opts(opts)
+    @fetch_opts ++ [recv_timeout: opts[:timeout]] ++ opts[:modifier].opts(opts)
   end
 end

lib/crawler/options.ex

@@ -17,7 +17,7 @@ defmodule Crawler.Options do
   @modifier   Crawler.Fetcher.Modifier
   @scraper    Crawler.Scraper
   @parser     Crawler.Parser
-  @encode     false
+  @encode_uri false
 
   @doc """
   Assigns default option values.