-
Notifications
You must be signed in to change notification settings - Fork 109
/
engine.ex
120 lines (95 loc) · 3.16 KB
/
engine.ex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
defmodule Crawly.Engine do
@moduledoc """
Crawly Engine - process responsible for starting and stopping spiders.
Stores all currently running spiders.
"""
require Logger
use GenServer
@type t :: %__MODULE__{started_spiders: started_spiders()}
@type started_spiders() :: %{optional(module()) => identifier()}
defstruct started_spiders: %{}
@spec start_spider(module()) ::
:ok
| {:error, :spider_already_started}
| {:error, :atom}
def start_spider(spider_name) do
GenServer.call(__MODULE__, {:start_spider, spider_name})
end
@spec get_manager(module()) ::
pid() | {:error, :spider_not_found}
def get_manager(spider_name) do
case Map.fetch(running_spiders(), spider_name) do
:error ->
{:error, :spider_not_found}
{:ok, pid_sup} ->
Supervisor.which_children(pid_sup)
|> Enum.find(&({Crawly.Manager, _, :worker, [Crawly.Manager]} = &1))
|> case do
nil -> {:error, :spider_not_found}
{_, pid, :worker, _} -> pid
end
end
end
@spec stop_spider(module(), reason) :: result
when reason: :itemcount_limit | :itemcount_timeout | atom(),
result: :ok | {:error, :spider_not_running}
def stop_spider(spider_name, reason \\ :ignore) do
case Crawly.Utils.get_settings(:on_spider_closed_callback, spider_name) do
nil -> :ignore
fun -> apply(fun, [reason])
end
GenServer.call(__MODULE__, {:stop_spider, spider_name})
end
@spec running_spiders() :: started_spiders()
def running_spiders() do
GenServer.call(__MODULE__, :running_spiders)
end
def start_link() do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
@spec init(any) :: {:ok, __MODULE__.t()}
def init(_args) do
{:ok, %Crawly.Engine{}}
end
def handle_call({:get_manager, spider_name}, _, state) do
pid =
case Map.get(state.started_spiders, spider_name) do
nil ->
{:error, :spider_not_found}
pid ->
pid
end
{:reply, pid, state}
end
def handle_call(:running_spiders, _from, state) do
{:reply, state.started_spiders, state}
end
def handle_call({:start_spider, spider_name}, _form, state) do
result =
case Map.get(state.started_spiders, spider_name) do
nil ->
Crawly.EngineSup.start_spider(spider_name)
_ ->
{:error, :spider_already_started}
end
{msg, new_started_spiders} =
case result do
{:ok, pid} ->
{:ok, Map.put(state.started_spiders, spider_name, pid)}
{:error, _} = err ->
{err, state.started_spiders}
end
{:reply, msg, %Crawly.Engine{state | started_spiders: new_started_spiders}}
end
def handle_call({:stop_spider, spider_name}, _form, state) do
{msg, new_started_spiders} =
case Map.pop(state.started_spiders, spider_name) do
{nil, _} ->
{{:error, :spider_not_running}, state.started_spiders}
{pid, new_started_spiders} ->
Crawly.EngineSup.stop_spider(pid)
{:ok, new_started_spiders}
end
{:reply, msg, %Crawly.Engine{state | started_spiders: new_started_spiders}}
end
end