Merge 58b91aa into 9b39d31

assets/css/phoenix.css

@@ -199,3 +199,22 @@ iframe {
     font-size: medium;
     color: black;
 }
+
+form.search{
+    width: 100%;
+}
+
+form.search input{
+    float: left;
+    margin: 0px;
+}
+
+form.search button{
+    float: right;
+}
+
+form.search .help_text{
+    font-size: smaller;
+    float: left;
+    margin-left: 2px;
+}

lib/crawly_ui/manager.ex

@@ -344,29 +344,55 @@ defmodule CrawlyUI.Manager do
   def list_items(job_id), do: list_items(job_id, [])
 
   def list_items(job_id, params) do
+    query = Item |> where([i], i.job_id == ^job_id)
+
     query =
       case Keyword.get(params, :search) do
         nil ->
-          Item
-          |> where([i], i.job_id == ^job_id)
-
-        search ->
-          case String.contains?(search, ":") do
-            true ->
-              [key, value] = String.split(search, ":")
-              value = "%#{String.trim(value)}%"
-
-              Item
-              |> where([i], i.job_id == ^job_id)
-              |> where([i], fragment("data->>? ILIKE ?", ^key, ^value))
-
-            false ->
-              Item
-              |> where([i], i.job_id == ^job_id)
+          query
+
+        search_string ->
+          case search(search_string) do
+            {:error, :parse_error} ->
+              query
+
+            ecto_fragment ->
+              query
+              |> where([i], ^ecto_fragment)
           end
       end
 
-    query |> order_by(desc: :inserted_at) |> Repo.paginate(params)
+    query
+    |> order_by(desc: :inserted_at)
+    |> Repo.paginate(params)
+  end
+
+  def parse_search_string(search_string) do
+    case CrawlyUI.QueryParser.query(search_string) do
+      {:ok, [], _rest, _map, _, _} -> {:error, :parse_error}
+      {:ok, tokens, _rest, _map, _, _} -> {:ok, tokens}
+    end
+  end
+
+  # Return an ECTO fragement for the given search query
+  def search(search_string) do
+    with {:ok, tokens} <- parse_search_string(search_string),
+         tokens = Enum.map(tokens, &String.trim(&1)) do
+      # take first two elements from the list
+      [key, value | rest] = tokens
+
+      Enum.reduce(
+        Enum.chunk_every(rest, 3),
+        dynamic(fragment("data->>? ILIKE ?", ^key, ^value)),
+        fn
+          ["||", key, value], acc ->
+            dynamic(^acc or fragment("data->>? ILIKE ?", ^key, ^value))
+
+          ["&&", key, value], acc ->
+            dynamic(^acc and fragment("data->>? ILIKE ?", ^key, ^value))
+        end
+      )
+    end
   end
 
   @doc """

lib/crawly_ui/query_parser.ex

@@ -0,0 +1,16 @@
+defmodule CrawlyUI.QueryParser do
+  import NimbleParsec
+
+  key = ascii_string([?a..?z, ?A..?Z, ?\s, ?\t], min: 1)
+  value = ascii_string([?a..?z, ?A..?Z, ?0..?9, ?\s, ?\t, ?%], min: 1)
+
+  or_ = string("||")
+  and_ = string("&&")
+
+  defcombinatorp(:expr, key |> ignore(string(":")) |> concat(value))
+  defcombinatorp(:operator, choice([or_, and_]))
+
+  defcombinatorp(:term, concat(parsec(:expr), optional(parsec(:operator))))
+
+  defparsec(:query, repeat(parsec(:term)))
+end

lib/crawly_ui_web/live/item_live.ex

@@ -100,7 +100,11 @@ defmodule CrawlyUIWeb.ItemLive do
 
     {:noreply,
      push_redirect(socket,
-       to: CrawlyUIWeb.Router.Helpers.item_path(socket, :index, job_id, page: page)
+       to:
+         CrawlyUIWeb.Router.Helpers.item_path(socket, :index, job_id,
+           page: page,
+           search: socket.assigns.search
+         )
      )}
   end
 

lib/crawly_ui_web/templates/item/index.html.leex

@@ -1,13 +1,23 @@
 <h1>Items</h1>
 
 <!-- The form -->
-<form phx-submit="search">
+<form phx-submit="search" class="search">
+
   <%= if @search do %>
-  <div class="column"><input type="text" placeholder="<%= @search %>" name="search"></div>
+  <div class="row">
+      <div class="column column-90"><input type="text" value="<%= @search %>" name="search" autocomplete="off"></div>
+      <div class="column column-offset-0">(*)<button type="submit"><i class="fa fa-search"></i></button></div>
+  </div>
   <% else %>
-  <div class="column"><input type="text" placeholder="Search" name="search"></div>
+  <div class="row">
+      <div class="column column-90">
+          <input type="text" placeholder="Search" name="search" autocomplete="off">
+          <div class="help_text">(*) <i>"field_name:field_value"</i> or more complex expressions <i>"title:%pi% && color:blue || color:green";</i> </div>
+      </div>
+      <div class="column column-offset-0"><button type="submit"><i class="fa fa-search"></i></button></div>
+  </div>
   <% end %>
-  <button type="submit"><i class="fa fa-search"></i></button>
+
 </form>
 <%= for item <- @rows do %>
 <table class="w3-table-all card">

mix.exs

@@ -43,7 +43,7 @@ defmodule CrawlyUI.MixProject do
       {:phoenix, "~> 1.5.4"},
       {:phoenix_pubsub, "~> 2.0"},
       {:phoenix_ecto, "~> 4.0"},
-      {:ecto_sql, "~> 3.1"},
+      {:ecto_sql, "~> 3.4"},
       {:postgrex, ">= 0.0.0"},
       {:phoenix_html, "~> 2.11"},
       {:phoenix_live_reload, "~> 1.2", only: :dev},
@@ -54,6 +54,7 @@ defmodule CrawlyUI.MixProject do
       {:timex, "~> 3.5"},
       {:phoenix_live_view, "~> 0.14.4"},
       {:scrivener_ecto, "~> 2.5.0"},
+      {:nimble_parsec, "~> 1.0"},
       {:floki, ">= 0.0.0", only: :test},
       {:excoveralls, "~> 0.10", only: :test},
       {:mock, "~> 0.3.0", only: :test}

mix.lock

@@ -23,6 +23,7 @@
   "mime": {:hex, :mime, "1.4.0", "5066f14944b470286146047d2f73518cf5cca82f8e4815cf35d196b58cf07c47", [:mix], [], "hexpm", "75fa42c4228ea9a23f70f123c74ba7cece6a03b1fd474fe13f6a7a85c6ea4ff6"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
   "mock": {:hex, :mock, "0.3.5", "feb81f52b8dcf0a0d65001d2fec459f6b6a8c22562d94a965862f6cc066b5431", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "6fae404799408300f863550392635d8f7e3da6b71abdd5c393faf41b131c8728"},
+  "nimble_parsec": {:hex, :nimble_parsec, "1.0.0", "54840f7a89aa3443c9280056742c329259d02787ef1da651f8591706518ceec6", [:mix], [], "hexpm", "6611313377408b6f19d4eddb1a53a342c70e336a21e9d09c38a9230775f1150d"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
   "phoenix": {:hex, :phoenix, "1.5.4", "0fca9ce7e960f9498d6315e41fcd0c80bfa6fbeb5fa3255b830c67fdfb7e703f", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4e516d131fde87b568abd62e1b14aa07ba7d5edfd230bab4e25cc9dedbb39135"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.2.0", "4ac3300a22240a37ed54dfe6c0be1b5623304385d1a2c210a70f011d9e7af7ac", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 2.15", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "59e7e2a550d7ea082a665c0fc29485f06f55d1a51dd02f513aafdb9d16fc72c4"},

test/crawly_ui/manager_test.exs

@@ -1,6 +1,8 @@
 defmodule CrawlyUi.ManagerTest do
   use CrawlyUI.DataCase
 
+  import Ecto.Query, warn: false
+
   alias CrawlyUI.Repo
 
   alias CrawlyUI.Manager
@@ -386,6 +388,40 @@ defmodule CrawlyUi.ManagerTest do
       job = insert_job()
       assert %{entries: []} = Manager.list_items(job.id, search: "id:1")
     end
+
+    test "search can filter items" do
+      job = insert_job(%{inserted_at: inserted_at(6 * 60)})
+
+      # Inserting items
+      insert_item(job.id, inserted_at(60 * 1), %{"id" => 1, "title" => "chair", "color" => "blue"})
+
+      insert_item(job.id, inserted_at(60 * 2), %{"id" => 2, "title" => "chair", "color" => "red"})
+      insert_item(job.id, inserted_at(60 * 3), %{"id" => 3, "title" => "sofa", "color" => "red"})
+
+      assert %{entries: [%Item{data: %{"id" => 1}}]} = Manager.list_items(job.id, search: "id:1")
+
+      assert %{entries: [%Item{data: %{"id" => 3}}]} =
+               Manager.list_items(job.id, search: "title:sofa")
+
+      assert %{entries: [%Item{data: %{"id" => 2}}]} =
+               Manager.list_items(job.id, search: "title:chair && color:red")
+    end
+
+    test "search can support wildcards" do
+      job = insert_job(%{inserted_at: inserted_at(6 * 60)})
+
+      # Inserting items
+      insert_item(job.id, inserted_at(60 * 1), %{"id" => 1, "title" => "chair", "color" => "blue"})
+
+      insert_item(job.id, inserted_at(60 * 2), %{"id" => 2, "title" => "chair2", "color" => "red"})
+
+      insert_item(job.id, inserted_at(60 * 3), %{"id" => 3, "title" => "sofa", "color" => "red"})
+
+      result = Manager.list_items(job.id, search: "title:chair%")
+      data = Enum.map(result.entries, fn item -> item.data end)
+
+      assert [%{"title" => "chair"}, %{"title" => "chair2"}] = data
+    end
   end
 
   test "get_item!/1" do

test/crawly_ui/query_parser_test.exs

@@ -0,0 +1,32 @@
+defmodule CrawlyUi.QueryParserTest do
+  use ExUnit.Case
+
+  test "parse simple key:value queries" do
+    assert {:ok, ["title", "name"], "", %{}, {1, 0}, 10} ==
+             CrawlyUI.QueryParser.query("title:name")
+  end
+
+  test "parse queries with AND" do
+    result = CrawlyUI.QueryParser.query("title:name && title:chair")
+    assert {:ok, ["title", "name ", "&&", " title", "chair"], "", %{}, {1, 0}, 25} == result
+  end
+
+  test "parse queries with OR" do
+    result = CrawlyUI.QueryParser.query("title:name || title:chair")
+    assert {:ok, ["title", "name ", "||", " title", "chair"], "", %{}, {1, 0}, 25} == result
+  end
+
+  test "parse queries with OR and AND" do
+    result = CrawlyUI.QueryParser.query("title:name && price:6 || title:chair")
+
+    assert {:ok, ["title", "name ", "&&", " price", "6 ", "||", " title", "chair"], "", %{},
+            {1, 0}, 36} == result
+  end
+
+  test "can parse incomplete queries" do
+    assert {:ok, [], "title", %{}, {1, 0}, 0} == CrawlyUI.QueryParser.query("title")
+    assert {:ok, [], "title::wow", %{}, {1, 0}, 0} == CrawlyUI.QueryParser.query("title::wow")
+    assert {:ok, [], "", %{}, {1, 0}, 0} == CrawlyUI.QueryParser.query("")
+    assert {:ok, [], "&& test:one", %{}, {1, 0}, 0} == CrawlyUI.QueryParser.query("&& test:one")
+  end
+end

test/crawly_ui_web/live/item_live_test.exs

@@ -71,13 +71,15 @@ defmodule CrawlyUIWeb.ItemLiveTest do
   test "handle empty search string", %{conn: conn, job_id: job_id} do
     {:ok, _view, html} = live(conn, "/jobs/#{job_id}/items?search=")
 
-    assert html =~ "<input type=\"text\" placeholder=\"Search\" name=\"search\"/>"
+    assert html =~
+             "<input type=\"text\" placeholder=\"Search\" name=\"search\" autocomplete=\"off\"/>"
   end
 
   test "handle valid search string", %{conn: conn, job_id: job_id} do
     {:ok, _view, html} = live(conn, "/jobs/#{job_id}/items?search=location%3ACanada")
 
-    assert html =~ "<input type=\"text\" placeholder=\"location:Canada\" name=\"search\"/>"
+    assert html =~
+             "<input type=\"text\" value=\"location:Canada\" name=\"search\" autocomplete=\"off\"/>"
   end
 
   test "update view with search result", %{conn: conn, job_id: job_id, item_id: item_id} do
@@ -146,7 +148,7 @@ defmodule CrawlyUIWeb.ItemLiveTest do
     {:ok, new_view, _html} =
       view
       |> render_click(:goto_page, %{"page" => "2"})
-      |> follow_redirect(conn, "/jobs/#{job_id}/items?page=2")
+      |> follow_redirect(conn, "/jobs/#{job_id}/items?page=2&search=")
 
     refute render(new_view) =~ "Discovery time: #{item_2.inserted_at}"
     assert render(new_view) =~ "Discovery time: #{item_1.inserted_at}"

test/crawly_ui_web/views/item_view_test.exs

@@ -62,14 +62,14 @@ defmodule CrawlyUIWeb.ItemViewTest do
       params = index_params(%{"field" => "value"}, nil)
 
       assert render_to_string(CrawlyUIWeb.ItemView, "index.html", params) =~
-               "<div class=\"column\"><input type=\"text\" placeholder=\"Search\" name=\"search\"></div>"
+               "<input type=\"text\" placeholder=\"Search\" name=\"search\" autocomplete=\"off\">"
     end
 
     test "with search param" do
       params = index_params(%{"field" => "value"}, "search string")
 
       assert render_to_string(CrawlyUIWeb.ItemView, "index.html", params) =~
-               "<div class=\"column\"><input type=\"text\" placeholder=\"search string\" name=\"search\"></div>"
+               "<input type=\"text\" value=\"search string\" name=\"search\" autocomplete=\"off\">"
     end
   end