Merge e5f93e7 into c95b798

elixir-crawly · Apr 10, 2020 · f399860 · f399860
2 parents c95b798 + e5f93e7
commit f399860
Show file tree

Hide file tree

Showing 5 changed files with 6 additions and 232 deletions.
diff --git a/README.md b/README.md
@@ -74,13 +74,13 @@ historical archival.
      middlewares: [
        Crawly.Middlewares.DomainFilter,
        Crawly.Middlewares.UniqueRequest,
-       Crawly.Middlewares.UserAgent
+       {Crawly.Middlewares.UserAgent, user_agents: ["Crawly Bot"]}
      ],
      pipelines: [
        {Crawly.Pipelines.Validate, fields: [:url, :title]},
        {Crawly.Pipelines.DuplicatesFilter, item_id: :title},
        Crawly.Pipelines.JSONEncoder,
-       {Crawly.Pipelines.WriteToFile, extension: "jl", folder: "/tmp"} # NEW IN 0.7.0
+       {Crawly.Pipelines.WriteToFile, extension: "jl", folder: "/tmp"}
       ],
       port: 4001
    ```
@@ -108,7 +108,7 @@ You can read more here:
 
 1. [x] Pluggable HTTP client
 2. [x] Retries support
-3. [ ] Cookies support
+3. [x] Cookies support
 4. [x] XPath support - can be actually done with meeseeks
 5. [ ] Project generators (spiders)
 6. [ ] UI for jobs management

diff --git a/documentation/introduction.md b/documentation/introduction.md
diff --git a/documentation/quickstart.md b/documentation/quickstart.md
diff --git a/lib/crawly/manager.ex b/lib/crawly/manager.ex
@@ -111,7 +111,7 @@ defmodule Crawly.Manager do
 
       maybe_stop_spider_by_timeout(
         state.name,
-        items_count,
+      delta,
         closespider_timeout_limit
       )
 

diff --git a/mix.exs b/mix.exs
@@ -101,13 +101,12 @@ defmodule Crawly.Mixfile do
 
   defp extras do
     [
-      "documentation/introduction.md",
-      "documentation/quickstart.md",
       "documentation/tutorial.md",
       "documentation/basic_concepts.md",
       "documentation/configuration.md",
       "documentation/http_api.md",
-      "documentation/ethical_aspects.md"
+      "documentation/ethical_aspects.md",
+      "README.md": [title: "Introduction", file: "README.md"]
     ]
   end
 end