fixed failing test, added mock for settings override in mocked spider

elixir-crawly · Nov 2, 2020 · 7944ec2 · 7944ec2
1 parent 1fc13ad
commit 7944ec2
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 19 deletions.
diff --git a/lib/crawly.ex b/lib/crawly.ex
@@ -15,10 +15,17 @@ defmodule Crawly do
 
 
   """
+  @type with_opt :: {:with, nil | module()}
+  @type request_opt :: {:request_options, list(Crawly.Request.option())}
+  @type headers_opt :: {:headers, list(Crawly.Request.header())}
 
   @spec fetch(url, opts) :: HTTPoison.Response.t()
         when url: binary(),
-             opts: list()
+             opts: [
+               with_opt
+               | request_opt
+               | headers_opt
+             ]
   def fetch(url, opts \\ []) do
     opts = Enum.into(opts, %{with: nil, request_options: [], headers: []})
 
@@ -68,13 +75,18 @@ defmodule Crawly do
                  opts[:with]
                ),
              items <- Map.get(parsed_result, :items, []),
-             pipeline_result <-
-               Enum.reduce(items, [], fn item, acc ->
-                 {piped, _state} = Crawly.Utils.pipe(pipelines, item, %{})
-
-                 [acc | piped]
+             {pipeline_result, pipeline_state} <-
+               Enum.reduce(items, {[], %{}}, fn item, {acc, state} ->
+                 {piped, state} = Crawly.Utils.pipe(pipelines, item, state)
+
+                 if piped == false do
+                   # dropped
+                   {acc, state}
+                 else
+                   {[piped | acc], state}
+                 end
                end) do
-          {response, parsed_result, pipeline_result}
+          {response, parsed_result, pipeline_result, pipeline_state}
         end
     end
   end

diff --git a/test/crawly_test.exs b/test/crawly_test.exs
@@ -3,37 +3,44 @@ defmodule CrawlyTest do
   doctest Crawly
 
   setup do
-    :meck.new(CrawlyTestSpider)
+    :meck.new(CrawlyTestSpider, [:non_strict])
 
-    :meck.expect(CrawlyTestSpider, :parse_items, fn resp ->
+    :meck.expect(CrawlyTestSpider, :parse_item, fn _resp ->
       %{
-        items: ["hello"],
+        items: [%{content: "hello"}],
         requests: [
           Crawly.Utils.request_from_url("https://www.example.com/test")
         ]
       }
     end)
 
+    :meck.expect(CrawlyTestSpider, :override_settings, fn ->
+      [pipelines: [Crawly.Pipelines.JSONEncoder]]
+    end)
+
     on_exit(fn ->
-      :meck.unload(CrawlyTestSpider)
+      :meck.unload()
     end)
+
+    {:ok, spider_module: CrawlyTestSpider}
   end
 
   test "fetch/1 is able to fetch a given url using global config, returns a response" do
     assert %HTTPoison.Response{} = Crawly.fetch("https://example.com")
   end
 
-  test "fetch/2 with :with option provided returns the response, parsed_item result, and processed ParsedItems" do
-    assert {%HTTPoison.Response{}, parsed_items_res, parsed_items} =
-             Crawly.fetch("http://example.com", with: CrawlyTestSpider)
+  test "fetch/2 with :with option provided returns the response, parsed_item result, and processed ParsedItems",
+       %{spider_module: spider_module} do
+    assert {%HTTPoison.Response{}, parsed_item_res, parsed_items,
+            pipeline_state} =
+             Crawly.fetch("http://example.com", with: spider_module)
 
     assert %{
-             items: items,
+             items: [_],
              requests: requests
-           } = parsed_items_res
+           } = parsed_item_res
 
-    assert is_list(parsed_items)
-    assert length(parsed_items) == 1
-    assert ["hello"] = parsed_items
+    assert [encoded] = parsed_items
+    assert encoded =~ "hello"
   end
 end