Add scraping one day condition

open-contracting · Nov 3, 2020 · ceef349 · ceef349
1 parent 802d822
commit ceef349
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/kingfisher_scrapy/base_spider.py b/kingfisher_scrapy/base_spider.py
@@ -444,6 +444,9 @@ def next_link(self, response, **kwargs):
         if url:
             return self.build_request(url, formatter=self.next_page_formatter, **kwargs)
 
+        if self.from_date and self.until_date and (self.until_date - self.from_date).days <= 1:
+            return
+
         if response.meta['depth'] == 0:
             raise MissingNextLinkError(f'next link not found on the first page: {response.url}')