Simplified logging for spiders

lib/crawly/data_storage/data_storage.ex

@@ -55,42 +55,23 @@ defmodule Crawly.DataStorage do
   def handle_call({:store, spider, item}, _from, state) do
     %{workers: workers} = state
 
-    {pid, new_workers} =
+    message =
       case Map.get(workers, spider) do
         nil ->
-          {:ok, pid} =
-            DynamicSupervisor.start_child(
-              Crawly.DataStorage.WorkersSup,
-              {Crawly.DataStorage.Worker, [spider_name: spider]}
-            )
-
-          {pid, Map.put(workers, spider, pid)}
+          {:error, :data_storage_worker_not_running}
 
         pid ->
-          {pid, workers}
+          Crawly.DataStorage.Worker.store(pid, item)
       end
 
-    Crawly.DataStorage.Worker.store(pid, item)
-    {:reply, :ok, %{state | workers: new_workers}}
+    {:reply, message, state}
   end
 
   def handle_call({:start_worker, spider_name, crawl_id}, _from, state) do
     {msg, new_state} =
       case Map.get(state.workers, spider_name) do
         nil ->
-          {:ok, pid} =
-            DynamicSupervisor.start_child(
-              Crawly.DataStorage.WorkersSup,
-              %{
-                id: :undefined,
-                restart: :temporary,
-                start:
-                  {Crawly.DataStorage.Worker, :start_link,
-                   [[spider_name: spider_name, crawl_id: crawl_id]]}
-              }
-            )
-
-          Process.monitor(pid)
+          pid = do_start_worker(spider_name, crawl_id)
 
           new_workers = Map.put(state.workers, spider_name, pid)
           new_spider_pids = Map.put(state.pid_spiders, pid, spider_name)
@@ -132,4 +113,21 @@ defmodule Crawly.DataStorage do
 
     {:noreply, new_state}
   end
+
+  defp do_start_worker(spider_name, crawl_id) do
+    {:ok, pid} =
+      DynamicSupervisor.start_child(
+        Crawly.DataStorage.WorkersSup,
+        %{
+          id: :undefined,
+          restart: :temporary,
+          start:
+            {Crawly.DataStorage.Worker, :start_link,
+             [[spider_name: spider_name, crawl_id: crawl_id]]}
+        }
+      )
+
+    Process.monitor(pid)
+    pid
+  end
 end

lib/crawly/data_storage/data_storage_worker.ex

@@ -32,6 +32,7 @@ defmodule Crawly.DataStorage.Worker do
   end
 
   def init(spider_name: spider_name, crawl_id: crawl_id) do
+    Logger.metadata(spider_name: spider_name, crawl_id: crawl_id)
     {:ok, %Worker{spider_name: spider_name, crawl_id: crawl_id}}
   end
 

lib/crawly/middlewares/auto_cookies_manager.ex

@@ -11,7 +11,6 @@ defmodule Crawly.Middlewares.AutoCookiesManager do
   ]
   ```
   """
-  require Logger
 
   def run(request, state) do
     known_cookies = Map.get(state, :cookies_manager_seen_cookies, MapSet.new())

lib/crawly/middlewares/domain_filter.ex

@@ -25,9 +25,7 @@ defmodule Crawly.Middlewares.DomainFilter do
     case host != nil and String.contains?(base_url, host) do
       false ->
         Logger.debug(
-          "Dropping request: #{inspect(request.url)} (domain filter)",
-          spider_name: state.spider_name,
-          crawl_id: state.crawl_id
+          "Dropping request: #{inspect(request.url)} (domain filter)"
         )
 
         {false, state}

lib/crawly/middlewares/robotstxt.ex

@@ -23,10 +23,7 @@ defmodule Crawly.Middlewares.RobotsTxt do
   def run(request, state, _opts \\ []) do
     case Gollum.crawlable?("Crawly", request.url) do
       :uncrawlable ->
-        Logger.debug("Dropping request: #{request.url} (robots.txt filter)",
-          spider_name: state.spider_name,
-          crawl_id: state.crawl_id
-        )
+        Logger.debug("Dropping request: #{request.url} (robots.txt filter)")
 
         {false, state}
 

lib/crawly/middlewares/unique_request.ex

@@ -24,9 +24,7 @@ defmodule Crawly.Middlewares.UniqueRequest do
 
       _ ->
         Logger.debug(
-          "Dropping request: #{request.url}, as it's already processed",
-          spider_name: state.spider_name,
-          crawl_id: state.crawl_id
+          "Dropping request: #{request.url}, as it's already processed"
         )
 
         {false, state}

lib/crawly/middlewares/user_agent.ex

@@ -16,7 +16,6 @@ defmodule Crawly.Middlewares.UserAgent do
   ]
   ```
   """
-  require Logger
 
   def run(request, state, opts \\ []) do
     opts = Enum.into(opts, %{user_agents: nil})

lib/crawly/pipelines/csv_encoder.ex

@@ -22,9 +22,7 @@ defmodule Crawly.Pipelines.CSVEncoder do
     case opts[:fields] do
       fields when fields in [nil, []] ->
         Logger.error(
-          "Dropping item: #{inspect(item)}. Reason: No fields declared for CSVEncoder",
-          spider_name: state.spider_name,
-          crawl_id: state.crawl_id
+          "Dropping item: #{inspect(item)}. Reason: No fields declared for CSVEncoder"
         )
 
         {false, state}

lib/crawly/pipelines/duplicates_filter.ex

@@ -40,9 +40,7 @@ defmodule Crawly.Pipelines.DuplicatesFilter do
       nil ->
         Logger.info(
           "Duplicates filter pipeline is inactive, item_id option is required
-          to make it operational.",
-          spider_name: state.spider_name,
-          crawl_id: state.crawl_id
+          to make it operational."
         )
 
         {item, state}

lib/crawly/pipelines/json_encoder.ex

@@ -30,9 +30,7 @@ defmodule Crawly.Pipelines.JSONEncoder do
       {:error, reason} ->
         Logger.error(
           "Could not encode the following item: #{inspect(item)} into json,
-          reason: #{inspect(reason)}",
-          spider_name: state.spider_name,
-          crawl_id: state.crawl_id
+          reason: #{inspect(reason)}"
         )
 
         {false, state}

lib/crawly/pipelines/validate.ex

@@ -48,9 +48,7 @@ defmodule Crawly.Pipelines.Validate do
 
       _ ->
         Logger.info(
-          "Dropping item: #{inspect(item)}. Reason: missing required fields",
-          spider_name: state.spider_name,
-          crawl_id: state.crawl_id
+          "Dropping item: #{inspect(item)}. Reason: missing required fields"
         )
 
         {false, state}

lib/crawly/requests_storage/requests_storage_worker.ex

@@ -52,12 +52,12 @@ defmodule Crawly.RequestsStorage.Worker do
   end
 
   def start_link(spider_name, crawl_id) do
-    Logger.debug("Starting requests storage worker for #{spider_name}...")
-
     GenServer.start_link(__MODULE__, [spider_name, crawl_id])
   end
 
   def init([spider_name, crawl_id]) do
+    Logger.metadata(spider_name: spider_name, crawl_id: crawl_id)
+    Logger.debug("Starting requests storage worker for #{spider_name}...")
     {:ok, %Worker{requests: [], spider_name: spider_name, crawl_id: crawl_id}}
   end
 

lib/crawly/worker.ex

@@ -20,6 +20,7 @@ defmodule Crawly.Worker do
   end
 
   def init(spider_name: spider_name, crawl_id: crawl_id) do
+    Logger.metadata(crawl_id: crawl_id, spider_name: spider_name)
     Crawly.Utils.send_after(self(), :work, 0)
 
     {:ok,
@@ -54,9 +55,7 @@ defmodule Crawly.Worker do
                 "Crawly worker could not process the request to #{
                   inspect(request.url)
                 }
-                  reason: #{inspect(reason)}",
-                spider_name: state.spider_name,
-                crawl_id: state.crawl_id
+                  reason: #{inspect(reason)}"
               )
 
               @default_backoff