Merge pull request #58 from oltarasenko/get_spider_loggs

Add support for spider logs

examples/config/config.exs

@@ -42,9 +42,10 @@ config :crawly,
 
 # tell logger to load a LoggerFileBackend processes
 config :logger,
-  backends: [:console, {LoggerFileBackend, :debug_log}]
-
-# configuration for the {LoggerFileBackend, :error_log} backend
-config :logger, :debug_log,
-  path: System.get_env("LOG_PATH", "/tmp/worker_debug.log"),
+  backends: [
+    :console,
+    {Crawly.Loggers.SendToUiBackend, :send_log_to_ui}
+  ],
   level: :debug
+
+config :logger, :send_log_to_ui, destination: {ui_node, CrawlyUI, :store_log}

examples/mix.exs

@@ -23,7 +23,7 @@ defmodule Shops.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
-      {:crawly, "~> 0.12.0"},
+      {:crawly, git: "https://github.com/oltarasenko/crawly.git"},
       {:floki, "~> 0.26.0"},
       {:logger_file_backend, "~> 0.0.11"},
       {:erlang_node_discovery, git: "https://github.com/oltarasenko/erlang-node-discovery"}

examples/mix.lock

@@ -3,7 +3,7 @@
   "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"},
   "cowlib": {:hex, :cowlib, "2.9.1", "61a6c7c50cf07fdd24b2f45b89500bb93b6686579b069a89f88cb211e1125c78", [:rebar3], [], "hexpm", "e4175dc240a70d996156160891e1c62238ede1729e45740bdd38064dad476170"},
-  "crawly": {:hex, :crawly, "0.12.0", "337d918fe9c39f2dfa06dfc7fd5bb774f790c4ac8ea4e18d6fbe757658cc1ac2", [:mix], [{:epipe, "~> 1.0", [hex: :epipe, repo: "hexpm", optional: false]}, {:gollum, "~> 0.4.0", [hex: :new_gollum, repo: "hexpm", optional: false]}, {:httpoison, "~> 1.7", [hex: :httpoison, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:poison, "~> 3.1", [hex: :poison, repo: "hexpm", optional: false]}, {:uuid, "~> 1.1", [hex: :uuid, repo: "hexpm", optional: false]}], "hexpm", "eed42d71099a2e7a0ba76585a86c7a2e44bc95f45a14b935719dd5ba46374cea"},
+  "crawly": {:git, "https://github.com/oltarasenko/crawly.git", "1892ce83b28ac224063d6b7b2962b2cd76198b9f", []},
   "epipe": {:hex, :epipe, "1.0.0", "9678d912c34b99dad9c20c4582d393d8101adfe96ebbe7af33b42f6619c96688", [:rebar3], [], "hexpm", "8a9d6ef41fa1e97b324726a93460fa712742a85a1957a02fdd9227912ce00ff6"},
   "erlang_node_discovery": {:git, "https://github.com/oltarasenko/erlang-node-discovery", "39ee515ede301260cca2ad913213b83262b1139c", []},
   "floki": {:hex, :floki, "0.26.0", "4df88977e2e357c6720e1b650f613444bfb48c5acfc6a0c646ab007d08ad13bf", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "e7b66ce7feef5518a9cd9fc7b52dd62a64028bd9cb6d6ad282a0f0fc90a4ae52"},
@@ -14,9 +14,8 @@
   "idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"},
   "logger_file_backend": {:hex, :logger_file_backend, "0.0.11", "3bbc5f31d3669e8d09d7a9443e86056fae7fc18e45c6f748c33b8c79a7e147a1", [:mix], [], "hexpm", "62be826f04644c62b0a2bc98a13e2e7ae52c0a4eda020f4c59d7287356d5e445"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
-  "mime": {:hex, :mime, "1.4.0", "5066f14944b470286146047d2f73518cf5cca82f8e4815cf35d196b58cf07c47", [:mix], [], "hexpm", "75fa42c4228ea9a23f70f123c74ba7cece6a03b1fd474fe13f6a7a85c6ea4ff6"},
+  "mime": {:hex, :mime, "1.5.0", "203ef35ef3389aae6d361918bf3f952fa17a09e8e43b5aa592b93eba05d0fb8d", [:mix], [], "hexpm", "55a94c0f552249fc1a3dd9cd2d3ab9de9d3c89b559c2bd01121f824834f24746"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
-  "new_gollum": {:hex, :new_gollum, "0.3.0", "a49164b63ea11b5c1586f8d0497bfa29978789bf3c939f11de140115a15f5ca9", [:mix], [{:httpoison, "~> 1.5.0", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "656c622e3f055b794851608373dd9d5f041d2320a38012fd8d959e6440d18706"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
   "plug": {:hex, :plug, "1.11.0", "f17217525597628298998bc3baed9f8ea1fa3f1160aa9871aee6df47a6e4d38e", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2d9c633f0499f9dc5c2fd069161af4e2e7756890b81adcbb2ceaa074e8308876"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.4.1", "779ba386c0915027f22e14a48919a9545714f849505fa15af2631a0d298abf0f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d72113b6dff7b37a7d9b2a5b68892808e3a9a752f2bf7e503240945385b70507"},

lib/crawly_ui.ex

@@ -23,6 +23,20 @@ defmodule CrawlyUI do
     end
   end
 
+  def store_log(job_tag, msg) do
+    case Manager.get_job_by_tag(job_tag) do
+      nil ->
+        Logger.error("Job was not found: #{job_tag}")
+        {:error, :job_not_found}
+
+      job ->
+        {:ok, _log} = Manager.create_log(%{job_id: job.id, message: msg})
+
+        Manager.update_job(job, %{state: "running"})
+        :ok
+    end
+  end
+
   def list_spiders(node) do
     :rpc.call(node, Crawly, :list_spiders, [])
   end

lib/crawly_ui/manager.ex

@@ -9,6 +9,7 @@ defmodule CrawlyUI.Manager do
   alias CrawlyUI.Manager.Job
   alias CrawlyUI.Manager.Item
   alias CrawlyUI.Manager.Spider
+  alias CrawlyUI.Manager.Log
 
   alias CrawlyUI.SpiderManager
 
@@ -541,6 +542,23 @@ defmodule CrawlyUI.Manager do
     end
   end
 
+  def create_log(attrs \\ %{}) do
+    %Log{}
+    |> Log.changeset(attrs)
+    |> Repo.insert()
+  end
+
+  @doc """
+  List generated spiders
+  """
+  def list_logs(job_id, params) do
+    query = Log |> where([i], i.job_id == ^job_id)
+
+    query
+    |> order_by(desc: :inserted_at)
+    |> Repo.paginate(params)
+  end
+
   # Convert all values in rules into Base encoded binaries (so they can be stored
   # in database)
   defp encode_rules(rules) do

lib/crawly_ui/manager/log.ex

@@ -0,0 +1,18 @@
+defmodule CrawlyUI.Manager.Log do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "logs" do
+    field :message, :string
+    field :job_id, :id
+
+    timestamps()
+  end
+
+  @doc false
+  def changeset(log, attrs) do
+    log
+    |> cast(attrs, [:job_id, :message])
+    |> validate_required([:job_id, :message])
+  end
+end

lib/crawly_ui_web/controllers/log_controller.ex

@@ -0,0 +1,17 @@
+defmodule CrawlyUIWeb.LogController do
+  use CrawlyUIWeb, :controller
+
+  alias CrawlyUI.Manager
+
+  def index(conn, %{"job_id" => id} = params) do
+    page = Manager.list_logs(id, params)
+
+    render(conn, "index.html",
+      id: id,
+      current_page: page.page_number,
+      items: page.entries,
+      page: page,
+      total_pages: page.total_pages
+    )
+  end
+end

lib/crawly_ui_web/router.ex

@@ -30,5 +30,7 @@ defmodule CrawlyUIWeb.Router do
     live "/jobs/:job_id/items/:id", ItemLive, :show
 
     get "/jobs/:job_id/export", ItemController, :export
+
+    get "/logs/:job_id/list", LogController, :index
   end
 end

lib/crawly_ui_web/templates/layout/app.html.eex

@@ -1,4 +1,3 @@
 <main role="main" class="container">
-    <p class="alert alert-info"><%= live_flash(@flash, :info) %></p>.
     <%= @inner_content %>
 </main>

lib/crawly_ui_web/templates/log/index.html.eex

@@ -0,0 +1,22 @@
+<h1>Logs</h1>
+<table>
+    <thead>
+        <tr>
+            <th>Timestamp</th>
+            <th>Message</th>
+        </tr>
+    </thead>
+    <tbody>
+        <%= for item <- @items do %>
+        <tr>
+            <td><%= item.inserted_at %></td>
+            <td><%= item.message %></td>
+        </tr>
+        <% end %>
+    </tbody>
+</table>
+<nav>
+    <ul class="pagination">
+        <li><a href="/logs/<%= @id %>/list/?page=<%= @current_page + 1 %>">Next page</a></li>
+    </ul>
+</nav>

lib/crawly_ui_web/views/job_view.ex

@@ -67,7 +67,7 @@ defmodule CrawlyUIWeb.JobView do
       <th>Spider</th>
       <th>Node</th>
       <th>Items</th>
-
+      <th>Logs</th>
       <th>State</th>
 
       <th>Start Time</th>
@@ -92,6 +92,9 @@ defmodule CrawlyUIWeb.JobView do
         <td>
           <a href="/jobs/#{job.id}/items">#{job.items_count}</a>
         </td>
+        <td>
+          <a href="/logs/#{job.id}/list">logs</a>
+        </td>
         <td>#{job.state}</td>
         <td>#{job.inserted_at}</td>
         <td>#{job.crawl_speed} items/min</td>

lib/crawly_ui_web/views/log_view.ex

@@ -0,0 +1,3 @@
+defmodule CrawlyUIWeb.LogView do
+  use CrawlyUIWeb, :view
+end