add links on next page

elixir-crawly · Feb 10, 2021 · c64c6bb · c64c6bb
1 parent 79341e7
commit c64c6bb
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/examples/lib/shops/spiders/allo_ua.ex b/examples/lib/shops/spiders/allo_ua.ex
@@ -38,13 +38,21 @@ defmodule Spiders.AlloUa do
     # Parse response body to document
     {:ok, document} = Floki.parse_document(response.body)
 
+    next_page =
+      document
+      |> Floki.find("div.pagination__next a.pagination__next__link")
+      |> Floki.attribute("href")
+      |> Floki.text()
+
     # Extract individual product page URLs
-    urls =
+    product_pages =
       document
       |> Floki.find("div.product-card__content")
       |> Floki.find("a.product-card__title")
       |> Floki.attribute("href")
 
+    urls = [next_page|product_pages]
+
     requests =
       urls
       |> Enum.uniq()