Ignore incorrect requests if they are passed from start_requests

elixir-crawly · Oct 16, 2020 · db15123 · db15123
1 parent 8c8e3f8
commit db15123
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/lib/crawly/manager.ex b/lib/crawly/manager.ex
@@ -68,8 +68,17 @@ defmodule Crawly.Manager do
     # Store start urls
     Enum.each(
       Keyword.get(init, :start_requests, []),
-      fn request ->
-        Crawly.RequestsStorage.store(spider_name, request)
+      fn
+        %Crawly.Request{} = request ->
+          Crawly.RequestsStorage.store(spider_name, request)
+
+        request ->
+          # We should not attempt to store something which is not a request
+          Logger.error(
+            "#{inspect(request)} does not seem to be a request. Ignoring."
+          )
+
+          :ignore
       end
     )
 

diff --git a/test/manager_test.exs b/test/manager_test.exs
@@ -192,7 +192,10 @@ defmodule Manager.StartRequestsTestSpider do
 
   def init() do
     [
-      start_requests: [Crawly.Request.new("https://www.example.com/blog.html")]
+      start_requests: [
+        Crawly.Request.new("https://www.example.com/blog.html"),
+        "Incorrect request"
+      ]
     ]
   end