diff --git a/lib/crawly/worker.ex b/lib/crawly/worker.ex index 4ca8c819..a531405d 100644 --- a/lib/crawly/worker.ex +++ b/lib/crawly/worker.ex @@ -43,16 +43,18 @@ defmodule Crawly.Worker do case :epipe.run(functions, {request, spider_name}) do {:error, _step, reason, _step_state} -> # TODO: Add retry logic - Logger.error( - fn -> - "Crawly worker could not process the request to #{inspect(request.url)} + Logger.error(fn -> + "Crawly worker could not process the request to #{ + inspect(request.url) + } reason: #{inspect(reason)}" - end) + end) + @default_backoff + {:ok, _result} -> @default_backoff end - end Process.send_after(self(), :work, new_backoff) @@ -66,6 +68,9 @@ defmodule Crawly.Worker do response: HTTPoison.Response.t(), result: {:ok, response, spider_name} | {:error, term()} defp get_response({request, spider_name}) do + # check if spider-level fetcher is set. Overrides the globally configured fetcher. + # if not set, log warning for explicit config preferred, get the globally-configured fetcher. Defaults to HTTPoison + case HTTPoison.get(request.url, request.headers, request.options) do {:ok, response} -> {:ok, {response, spider_name}}