Respect max_items_read parameter for FOLLOW pagination

holgerd77 · Jun 26, 2017 · a97ef00 · a97ef00
1 parent 87d5080
commit a97ef00
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/dynamic_scraper/spiders/django_spider.py b/dynamic_scraper/spiders/django_spider.py
@@ -756,7 +756,11 @@ def parse(self, response):
                         self.log("Value after: " + value, logging.DEBUG)
             else:
                 self.log("Item could not be read!", logging.ERROR)
-        if self.scraper.follow_pages_url_xpath:
+
+        mir_reached = False
+        if self.conf['MAX_ITEMS_READ'] and (self.conf['MAX_ITEMS_READ'] - self.items_read_count <= 0):
+            mir_reached = True
+        if self.scraper.follow_pages_url_xpath and not mir_reached:
             if not self.scraper.num_pages_follow or follow_page_num < self.scraper.num_pages_follow:
                 url = response.xpath(self.scraper.follow_pages_url_xpath).extract_first()
                 if url is not None: