37 spider management Crawly.Engine.start_all_spiders/0

lib/crawly/engine.ex

@@ -13,6 +13,9 @@ defmodule Crawly.Engine do
           known_spiders: [module()]
         }
   @type started_spiders() :: %{optional(module()) => identifier()}
+  @type list_spiders() :: [
+          %{name: module(), state: :stopped | :started, pid: identifier()}
+        ]
 
   @type spider_info() :: %{
           name: module(),
@@ -30,8 +33,12 @@ defmodule Crawly.Engine do
     GenServer.call(__MODULE__, {:start_spider, spider_name, crawl_id})
   end
 
-  @spec get_manager(module()) ::
-          pid() | {:error, :spider_not_found}
+  @spec start_all_spiders() :: :ok
+  def start_all_spiders() do
+    GenServer.call(__MODULE__, :start_all_spiders)
+  end
+
+  @spec get_manager(module()) :: pid() | {:error, :spider_not_found}
   def get_manager(spider_name) do
     case Map.fetch(running_spiders(), spider_name) do
       :error ->
@@ -154,6 +161,28 @@ defmodule Crawly.Engine do
     {:reply, msg, %Crawly.Engine{state | started_spiders: new_started_spiders}}
   end
 
+  def handle_call(:start_all_spiders, _form, state) do
+    stopped =
+      state.known_spiders
+      |> Enum.filter(fn s -> Map.has_key?(state.started_spiders, s) == false end)
+
+    # start all stopped spiders
+    new_started_spiders =
+      Enum.reduce(stopped, state.started_spiders, fn s, acc ->
+        Crawly.EngineSup.start_spider(s)
+        |> case do
+          {:ok, pid} ->
+            Map.put(acc, s, pid)
+
+          _ ->
+            acc
+        end
+      end)
+
+    # returns ok regardless of whether errors encountered
+    {:reply, :ok, %Crawly.Engine{state | started_spiders: new_started_spiders}}
+  end
+
   def handle_call({:stop_spider, spider_name}, _form, state) do
     {msg, new_started_spiders} =
       case Map.pop(state.started_spiders, spider_name) do

test/engine_test.exs

@@ -1,6 +1,16 @@
 defmodule EngineTest do
   use ExUnit.Case
 
+  setup do
+    stop_all()
+    on_exit(&stop_all/0)
+  end
+
+  defp stop_all do
+    Crawly.Utils.list_spiders()
+    |> Enum.each(fn s -> Crawly.Engine.stop_spider(s) end)
+  end
+
   test "list_known_spiders/0 lists all spiders and their current status in the engine" do
     Crawly.Engine.init([])
     Crawly.Engine.refresh_spider_list()
@@ -16,12 +26,21 @@ defmodule EngineTest do
              Crawly.Engine.list_known_spiders()
              |> Enum.find(fn s -> s.name == TestSpider end)
 
-    assert :started = started_status.status
+    assert started_status.status == :started
     assert started_status.pid
+  end
+
+  test "start_all_spiders/0 starts all spiders in the engine" do
+    assert :ok = Crawly.Engine.start_all_spiders()
+    infos = Crawly.Engine.list_known_spiders()
 
-    # stop spider
+    assert Enum.all?(infos, fn info ->
+             info.status == :started and not is_nil(info.pid)
+           end)
+
+    # stop spiders
     Crawly.Engine.stop_spider(TestSpider)
-    spiders = Crawly.Engine.list_known_spiders()
-    assert Enum.all?(spiders, fn s -> s.status == :stopped end)
+    infos = Crawly.Engine.list_known_spiders()
+    assert Enum.all?(infos, fn s -> s.status == :stopped end)
   end
 end