From ceef349e3dac80f6a4afcb667436f22b835d4e13 Mon Sep 17 00:00:00 2001 From: aguilerapy Date: Tue, 3 Nov 2020 15:51:47 -0300 Subject: [PATCH] Add scraping one day condition --- kingfisher_scrapy/base_spider.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kingfisher_scrapy/base_spider.py b/kingfisher_scrapy/base_spider.py index 592a77e2..46d78943 100644 --- a/kingfisher_scrapy/base_spider.py +++ b/kingfisher_scrapy/base_spider.py @@ -444,6 +444,9 @@ def next_link(self, response, **kwargs): if url: return self.build_request(url, formatter=self.next_page_formatter, **kwargs) + if self.from_date and self.until_date and (self.until_date - self.from_date).days <= 1: + return + if response.meta['depth'] == 0: raise MissingNextLinkError(f'next link not found on the first page: {response.url}')