Merge pull request #121 from maiphuong-van/fix-itemcount-timeout

[Fix] Do not close spider with :itemcount_timeout when closespider_timeout is set to :disabled

lib/crawly/manager.ex

@@ -162,7 +162,7 @@ defmodule Crawly.Manager do
   defp maybe_stop_spider_by_itemcount_limit(_, _, _), do: :ok
 
   defp maybe_stop_spider_by_timeout(spider_name, current, limit)
-       when current < limit do
+       when current < limit and is_integer(limit) do
     Logger.info("Stopping #{inspect(spider_name)}, itemcount timeout achieved")
 
     Crawly.Engine.stop_spider(spider_name, :itemcount_timeout)

test/manager_test.exs

@@ -4,7 +4,6 @@ defmodule ManagerTest do
   setup do
     Application.put_env(:crawly, :concurrent_requests_per_domain, 1)
     Application.put_env(:crawly, :closespider_itemcount, 10)
-    Application.put_env(:crawly, :concurrent_requests_per_domain, 1)
 
     :meck.expect(HTTPoison, :get, fn _, _, _ ->
       {:ok,
@@ -91,6 +90,20 @@ defmodule ManagerTest do
     assert %{} == Crawly.Engine.running_spiders()
   end
 
+  test "spider does not close after 1 minute when closespider timeout is disabled" do
+    Application.put_env(:crawly, :closespider_timeout, :disabled)
+    Application.put_env(:crawly, :manager_operations_timeout, 1_000)
+
+    :ok = Crawly.Engine.start_spider(Manager.TestSpider)
+
+    Process.sleep(1_001)
+
+    refute %{} == Crawly.Engine.running_spiders()
+
+    :ok = Crawly.Engine.stop_spider(Manager.TestSpider)
+    assert %{} == Crawly.Engine.running_spiders()
+  end
+
   test "Can't start already started spider" do
     :ok = Crawly.Engine.start_spider(Manager.TestSpider)