Improve spider logging

examples/mix.exs

@@ -23,7 +23,7 @@ defmodule Shops.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
-      {:crawly, git: "https://github.com/oltarasenko/crawly.git"},
+      {:crawly, git: "https://github.com/oltarasenko/crawly.git", branch: "send_logs_module"},
       {:floki, "~> 0.26.0"},
       {:logger_file_backend, "~> 0.0.11"},
       {:erlang_node_discovery, git: "https://github.com/oltarasenko/erlang-node-discovery"}

examples/mix.lock

@@ -3,7 +3,7 @@
   "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"},
   "cowlib": {:hex, :cowlib, "2.9.1", "61a6c7c50cf07fdd24b2f45b89500bb93b6686579b069a89f88cb211e1125c78", [:rebar3], [], "hexpm", "e4175dc240a70d996156160891e1c62238ede1729e45740bdd38064dad476170"},
-  "crawly": {:git, "https://github.com/oltarasenko/crawly.git", "1892ce83b28ac224063d6b7b2962b2cd76198b9f", []},
+  "crawly": {:git, "https://github.com/oltarasenko/crawly.git", "7b38b68c468819492bce8d051cff71e90167673e", [branch: "send_logs_module"]},
   "epipe": {:hex, :epipe, "1.0.0", "9678d912c34b99dad9c20c4582d393d8101adfe96ebbe7af33b42f6619c96688", [:rebar3], [], "hexpm", "8a9d6ef41fa1e97b324726a93460fa712742a85a1957a02fdd9227912ce00ff6"},
   "erlang_node_discovery": {:git, "https://github.com/oltarasenko/erlang-node-discovery", "39ee515ede301260cca2ad913213b83262b1139c", []},
   "floki": {:hex, :floki, "0.26.0", "4df88977e2e357c6720e1b650f613444bfb48c5acfc6a0c646ab007d08ad13bf", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "e7b66ce7feef5518a9cd9fc7b52dd62a64028bd9cb6d6ad282a0f0fc90a4ae52"},

lib/crawly_ui.ex

@@ -23,14 +23,14 @@ defmodule CrawlyUI do
     end
   end
 
-  def store_log(job_tag, msg) do
+  def store_log(job_tag, msg, mod) do
     case Manager.get_job_by_tag(job_tag) do
       nil ->
         Logger.error("Job was not found: #{job_tag}")
         {:error, :job_not_found}
 
       job ->
-        {:ok, _log} = Manager.create_log(%{job_id: job.id, message: msg})
+        {:ok, _log} = Manager.create_log(%{job_id: job.id, message: msg, mod: mod})
 
         Manager.update_job(job, %{state: "running"})
         :ok

lib/crawly_ui/manager.ex

@@ -551,9 +551,27 @@ defmodule CrawlyUI.Manager do
   @doc """
   List generated spiders
   """
-  def list_logs(job_id, params) do
+  def list_logs(job_id, params, filter) do
     query = Log |> where([i], i.job_id == ^job_id)
 
+    query =
+      case filter do
+        "all" ->
+          query
+
+        "workers" ->
+          query |> where([i], ilike(i.mod, "%worker%"))
+
+        "manager" ->
+          query |> where([i], ilike(i.mod, "%manager%"))
+
+        "requests" ->
+          query |> where([i], ilike(i.mod, "%middleware%"))
+
+        "items" ->
+          query |> where([i], ilike(i.mod, "%pipelines%"))
+      end
+
     query
     |> order_by(desc: :inserted_at)
     |> Repo.paginate(params)

lib/crawly_ui/manager/log.ex

@@ -5,14 +5,15 @@ defmodule CrawlyUI.Manager.Log do
   schema "logs" do
     field :message, :string
     field :job_id, :id
+    field :mod, :string
 
     timestamps()
   end
 
   @doc false
   def changeset(log, attrs) do
     log
-    |> cast(attrs, [:job_id, :message])
-    |> validate_required([:job_id, :message])
+    |> cast(attrs, [:job_id, :message, :mod])
+    |> validate_required([:job_id, :message, :mod])
   end
 end

lib/crawly_ui_web/controllers/log_controller.ex

@@ -4,10 +4,15 @@ defmodule CrawlyUIWeb.LogController do
   alias CrawlyUI.Manager
 
   def index(conn, %{"job_id" => id} = params) do
-    page = Manager.list_logs(id, params)
+    filter = Map.get(params, "logs_filter", "all")
+    filters = ["requests", "items", "workers", "manager"]
+    page = Manager.list_logs(id, params, filter)
 
     render(conn, "index.html",
       id: id,
+      extra_params: "logs_filter=#{filter}",
+      current_filter: filter,
+      filters: filters,
       current_page: page.page_number,
       items: page.entries,
       page: page,

lib/crawly_ui_web/helpers/pagination_helpers.ex

@@ -1,4 +1,41 @@
 defmodule CrawlyUIWeb.PaginationHelpers do
+  def scrivener_links(page, total_pages, link_params \\ nil) do
+    min =
+      case page - 5 <= 0 do
+        true -> 1
+        false -> page - 5
+      end
+
+    max =
+      case page + 5 >= total_pages do
+        true -> total_pages
+        false -> page + 5
+      end
+
+    pages =
+      Enum.map(
+        min..max,
+        fn
+          ^page ->
+            "<li class='active'><a href='#{build_pagination_url(page, link_params)}'>#{page}</a></li>"
+
+          x ->
+            "<li><a href='#{build_pagination_url(x, link_params)}'>#{x}</a></li>"
+        end
+      )
+
+    """
+    <nav>
+    <ul class="pagination">
+    #{pages}
+    </ul>
+    </nav>
+    """
+  end
+
+  defp build_pagination_url(page, nil), do: "?page=#{page}"
+  defp build_pagination_url(page, extra_params), do: "?page=#{page}&#{extra_params}"
+
   def pagination_links(page, total_pages) do
     goto_next = render_goto_next(page, total_pages)
     goto_prev = render_goto_prev(page, total_pages)

lib/crawly_ui_web/templates/log/index.html.eex

@@ -1,4 +1,16 @@
 <h1>Logs</h1>
+<!-- The form -->
+<form class="filter" method="GET" action="/logs/<%= @id %>/list">
+    <label for="logs_filter">Filter logs:</label>
+
+    <select name="logs_filter" id="logs_filter" onchange="this.form.submit()">
+        <option value="all">all</option>
+        <%= for filter <- @filters do %>
+        <option <%= if filter == @current_filter do %> selected <% end %> value="<%= filter %>"><%= filter %></option>
+        <% end %>
+    </select>
+</form>
+
 <table>
     <thead>
         <tr>
@@ -15,8 +27,5 @@
         <% end %>
     </tbody>
 </table>
-<nav>
-    <ul class="pagination">
-        <li><a href="/logs/<%= @id %>/list/?page=<%= @current_page + 1 %>">Next page</a></li>
-    </ul>
-</nav>
+
+<%= raw scrivener_links(@current_page, @total_pages, @extra_params) %>

priv/repo/migrations/20201216094712_log_model_add_module_field.exs

@@ -0,0 +1,9 @@
+defmodule CrawlyUI.Repo.Migrations.LogModelAddModuleField do
+  use Ecto.Migration
+
+  def change do
+    alter table(:logs) do
+      add :mod, :string
+    end
+  end
+end