diff --git a/lib/crawly/data_storage/data_storage.ex b/lib/crawly/data_storage/data_storage.ex index 3daa9f58..dc8f7e8e 100644 --- a/lib/crawly/data_storage/data_storage.ex +++ b/lib/crawly/data_storage/data_storage.ex @@ -43,7 +43,7 @@ defmodule Crawly.DataStorage do end def start_link([]) do - Logger.info("Starting data storage") + Logger.debug("Starting data storage") GenServer.start_link(__MODULE__, [], name: __MODULE__) end diff --git a/lib/crawly/manager.ex b/lib/crawly/manager.ex index 0d1e41cc..066d313b 100644 --- a/lib/crawly/manager.ex +++ b/lib/crawly/manager.ex @@ -32,7 +32,7 @@ defmodule Crawly.Manager do use GenServer def start_link(spider_name) do - Logger.info("Starting the manager for #{spider_name}") + Logger.debug("Starting the manager for #{spider_name}") GenServer.start_link(__MODULE__, spider_name) end diff --git a/lib/crawly/pipelines/csv_encoder.ex b/lib/crawly/pipelines/csv_encoder.ex index f0e73a3d..f4af184c 100644 --- a/lib/crawly/pipelines/csv_encoder.ex +++ b/lib/crawly/pipelines/csv_encoder.ex @@ -24,7 +24,7 @@ defmodule Crawly.Pipelines.CSVEncoder do :undefined -> # only for when both tuple and global config is not provided - Logger.info( + Logger.error( "Dropping item: #{inspect(item)}. Reason: No fields declared for CSVEncoder" ) diff --git a/lib/crawly/pipelines/duplicates_filter.ex b/lib/crawly/pipelines/duplicates_filter.ex index 6586d6c3..3c0a2003 100644 --- a/lib/crawly/pipelines/duplicates_filter.ex +++ b/lib/crawly/pipelines/duplicates_filter.ex @@ -60,7 +60,7 @@ defmodule Crawly.Pipelines.DuplicatesFilter do {item, new_state} true -> - Logger.info("[error] Duplicates filter, removed item: #{inspect(item)}") + Logger.debug("Duplicates filter dropped item: #{inspect(item)}") {false, state} end end diff --git a/lib/crawly/pipelines/json_encoder.ex b/lib/crawly/pipelines/json_encoder.ex index 5c0ada92..3ac63eb0 100644 --- a/lib/crawly/pipelines/json_encoder.ex +++ b/lib/crawly/pipelines/json_encoder.ex @@ -28,7 +28,7 @@ defmodule Crawly.Pipelines.JSONEncoder do {new_item, state} {:error, reason} -> - Logger.info( + Logger.error( "Could not encode the following item: #{inspect(item)} into json, reason: #{inspect(reason)}" ) diff --git a/lib/crawly/worker.ex b/lib/crawly/worker.ex index 91238814..af6cb5ac 100644 --- a/lib/crawly/worker.ex +++ b/lib/crawly/worker.ex @@ -42,8 +42,8 @@ defmodule Crawly.Worker do case :epipe.run(functions, {request, spider_name}) do {:error, _step, reason, _step_state} -> - # TODO: Add retry logic - Logger.error( + + Logger.debug( fn -> "Crawly worker could not process the request to #{ inspect(request.url) @@ -115,7 +115,7 @@ defmodule Crawly.Worker do error, reason -> stacktrace = :erlang.get_stacktrace() - Logger.error( + Logger.debug( "Could not parse item, error: #{inspect(error)}, reason: #{ inspect(reason) }, stacktrace: #{inspect(stacktrace)} @@ -164,7 +164,7 @@ defmodule Crawly.Worker do case retries <= max_retries do true -> - Logger.info("Request to #{request.url}, is scheduled for retry") + Logger.debug("Request to #{request.url}, is scheduled for retry") middlewares = request.middlewares -- ignored_middlewares @@ -176,7 +176,7 @@ defmodule Crawly.Worker do :ok = Crawly.RequestsStorage.store(spider, request) false -> - Logger.info("Dropping request to #{request.url}, (max retries)") + Logger.error("Dropping request to #{request.url}, (max retries)") :ok end