Merge pull request #47 from oltarasenko/avg_speed_fix

Avg speed fix

config/config.exs

@@ -30,11 +30,14 @@ config :phoenix, :json_library, Jason
 
 config :crawly_ui, CrawlyUI.Scheduler,
   jobs: [
-    # Every hour
-    {"* */1 * * *", {CrawlyUI.Manager, :update_job_status, []}},
+    # Every 5 mins
+    {"*/5 * * * *", {CrawlyUI.Manager, :update_job_status, []}},
 
-    # Every minute
-    {"*/4 * * * *", {CrawlyUI.Manager, :update_running_jobs, []}}
+    # Every 5 mins
+    {"*/5 * * * *", {CrawlyUI.Manager, :update_running_jobs, []}},
+
+    # Every 5 mins
+    {"*/5 * * * *", {CrawlyUI.Manager, :update_jobs_speed, []}}
   ]
 
 # Import environment specific config. This must remain at the bottom

config/prod.exs

@@ -13,6 +13,9 @@ config :crawly_ui, CrawlyUIWeb.Endpoint,
   server: true,
   url: [host: "example.com", port: 80]
 
+config :crawly_ui, CrawlyUIWeb.JobLive, update_interval: 20_000
+config :crawly_ui, CrawlyUIWeb.ItemLive, update_interval: 20_000
+
 # Do not print debug messages in production
 config :logger, level: :info
 

config/releases.exs

@@ -31,6 +31,9 @@ config :crawly_ui, CrawlyUIWeb.Endpoint,
   ],
   secret_key_base: secret_key_base
 
+config :crawly_ui, CrawlyUIWeb.JobLive, update_interval: 20_000
+config :crawly_ui, CrawlyUIWeb.ItemLive, update_interval: 20_000
+
 # ## Using releases (Elixir v1.9+)
 #
 # If you are doing OTP releases, you need to instruct Phoenix

lib/crawly_ui/manager.ex

@@ -11,6 +11,8 @@ defmodule CrawlyUI.Manager do
 
   alias CrawlyUI.SpiderManager
 
+  require Logger
+
   @job_abandoned_timeout 60 * 30
 
   @doc """
@@ -316,6 +318,27 @@ defmodule CrawlyUI.Manager do
     update_item_counts(jobs)
   end
 
+  @doc """
+  Update crawl speed, run time and item counts for all active jobs
+  """
+  def update_jobs_speed() do
+    Logger.info("Updating crawl speeds")
+
+    jobs =
+      Job
+      |> where([j], j.crawl_speed == 0 and j.state != "running")
+      |> Repo.all()
+
+    Enum.each(
+      jobs,
+      fn job ->
+        if job.run_time != 0 do
+          update_job(job, %{crawl_speed: (job.items_count / job.run_time) |> trunc()})
+        end
+      end
+    )
+  end
+
   @doc """
   Update crawl speed, run time and item counts for all jobs
   """

lib/crawly_ui_web/live/job_live.ex

@@ -37,19 +37,6 @@ defmodule CrawlyUIWeb.JobLive do
     {:noreply, socket}
   end
 
-  def handle_event("schedule", _, socket) do
-    {:noreply, push_redirect(socket, to: "/schedule")}
-  end
-
-  def handle_event("job_items", %{"id" => job_id}, socket) do
-    {:noreply,
-     push_redirect(socket, to: CrawlyUIWeb.Router.Helpers.item_path(socket, :index, job_id))}
-  end
-
-  def handle_event("list_all_jobs", _, socket) do
-    {:noreply, push_redirect(socket, to: "/all")}
-  end
-
   def handle_event("goto_page", %{"page" => page}, socket) do
     live_action = socket.assigns.live_action
 

lib/crawly_ui_web/templates/item/index.html.leex

@@ -7,15 +7,15 @@
   <div class="row">
       <div class="column column-90">
           <input type="text" value="<%= @search %>" name="search" autocomplete="off">
-          <div class="help_text">(*) <i>"field_name:field_value"</i> or more complex expressions <i>"title:%pi% && color:blue || color:green";</i> </div>
+          <div class="help_text">(*) <i>field_name:field_value</i>. Use % for wildcards, || and && to combine queries, e.g.: <i>title:%pi% && color:blue</i> </div>
       </div>
       <div class="column column-offset-0"><button type="submit"><i class="fa fa-search"></i></button></div>
   </div>
   <% else %>
   <div class="row">
       <div class="column column-90">
           <input type="text" placeholder="Search" name="search" autocomplete="off">
-          <div class="help_text">(*) <i>"field_name:field_value"</i> or more complex expressions <i>"title:%pi% && color:blue || color:green";</i> </div>
+          <div class="help_text">(*) General use case: <i>field_name:field_value</i>. Use % for wildcards, || and && to combine queries, e.g.: <i>title:%pi% && color:blue || color:green</i> </div>
       </div>
       <div class="column column-offset-0"><button type="submit"><i class="fa fa-search"></i></button></div>
   </div>

lib/crawly_ui_web/templates/job/index.html.leex

@@ -4,7 +4,7 @@
 <p>
 No jobs are currently running. You can schedule a new one by hitting Schedule button
 </p>
-<button phx-click=schedule>Schedule</button>
+<a href="/schedule" class="button button-black">Schedule</a>
 <% else %>
 <%= raw render_jobs_table(@rows, @live_action) %>
 <%= raw pagination_links(@page, @total_pages) %>
@@ -14,4 +14,4 @@ No jobs are currently running. You can schedule a new one by hitting Schedule bu
 <br>
 <h1>Recent Jobs</h1>
 <%= raw render_jobs_table(@recent_rows, @live_action) %>
-<button phx-click=list_all_jobs>See more</button>
+<a href="/all" class="button button-black">See more</a>

lib/crawly_ui_web/views/job_view.ex

@@ -90,7 +90,7 @@ defmodule CrawlyUIWeb.JobView do
         <td>#{render_spider_col(job.spider, live_action)}</td>
         <td>#{job.node}</td>
         <td>
-          <a href="#" phx-click="job_items" phx-value-id=#{job.id}>#{job.items_count}</a>
+          <a href="/jobs/#{job.id}/items">#{job.items_count}</a>
         </td>
         <td>#{job.state}</td>
         <td>#{job.inserted_at}</td>
@@ -106,7 +106,7 @@ defmodule CrawlyUIWeb.JobView do
   defp render_spider_col(spider, :spider), do: render_spider_name(spider)
 
   defp render_spider_col(spider, _) do
-    " <a href=\"#\" phx-click=\"show_spider\" phx-value-spider=#{spider}>#{
+    " <a href=\"\" phx-click=\"show_spider\" phx-value-spider=#{spider}>#{
       render_spider_name(spider)
     }</a></td>"
   end

test/crawly_ui_web/live/job_live_test.exs

@@ -26,30 +26,6 @@ defmodule CrawlyUIWeb.JobLiveTest do
     assert html =~ "Jobs"
   end
 
-  test "mount job view for showing all jobs", %{conn: conn} do
-    insert_job(%{state: "abandoned"})
-    {:ok, _view, html} = live(conn, "/all")
-    assert html =~ "Jobs"
-  end
-
-  test "redirect when click schedule", %{conn: conn} do
-    {:ok, view, _html} = live(conn, "/")
-    render_click(view, :schedule)
-    assert_redirect(view, "/schedule")
-  end
-
-  test "redirect when click on job's items", %{conn: conn} do
-    {:ok, view, _html} = live(conn, "/")
-    render_click(view, :job_items, %{"id" => "1"})
-    assert_redirect(view, "/jobs/1/items")
-  end
-
-  test "redirect when on list all jobs", %{conn: conn} do
-    {:ok, view, _html} = live(conn, "/")
-    render_click(view, :list_all_jobs)
-    assert_redirect(view, "/all")
-  end
-
   test "redirect to a spider's jobs", %{conn: conn} do
     {:ok, view, _html} = live(conn, "/")
     render_click(view, :show_spider, %{"spider" => "TestSpider"})