-
Notifications
You must be signed in to change notification settings - Fork 91
/
recorder.ex
42 lines (34 loc) · 943 Bytes
/
recorder.ex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
defmodule Crawler.Fetcher.Recorder do
@moduledoc """
Records information about each crawl for internal use.
"""
alias Crawler.Store
@doc """
Records information about each crawl for internal use.
## Examples
iex> Recorder.record(url: "url1", depth: 2)
{:ok, %{depth: 3, url: "url1"}}
iex> Recorder.record(url: "url2", depth: 2)
iex> Store.find("url2")
%Page{url: "url2"}
"""
def record(opts) do
with opts <- Enum.into(opts, %{}),
{:ok, _pid} <- store_url(opts),
opts <- store_url_depth(opts) do
{:ok, opts}
end
end
@doc """
Stores page data in `Crawler.Store.DB` for internal or external consumption.
"""
def store_page(body, opts) do
{:ok, Store.add_page_data(opts[:url], body, opts)}
end
defp store_url(opts) do
Store.add(opts[:url])
end
defp store_url_depth(opts) do
Map.replace!(opts, :depth, opts[:depth] + 1)
end
end