From 1a357d1e7d5fd58852f9a97313db87d0f8131742 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Wed, 7 Oct 2020 09:53:59 -0300 Subject: [PATCH] Update date required spiders --- kingfisher_scrapy/base_spider.py | 13 ++++--------- kingfisher_scrapy/spiders/paraguay_dncp_base.py | 6 +++--- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/kingfisher_scrapy/base_spider.py b/kingfisher_scrapy/base_spider.py index cb9632734..f1903a335 100644 --- a/kingfisher_scrapy/base_spider.py +++ b/kingfisher_scrapy/base_spider.py @@ -72,7 +72,7 @@ class BaseSpider(scrapy.Spider): date_format = 'date' # Set `date_required` to True in class attribute to always set the `from` and `until` date parameters. - # If `date_required` is true the attribute `default_from_date` should be set too. + # If `date_required` is True, the attribute `default_from_date` should be set too. date_required = False def __init__(self, sample=None, note=None, from_date=None, until_date=None, crawl_time=None, @@ -493,6 +493,9 @@ class PeriodicSpider(SimpleSpider): """ VALID_DATE_FORMATS = {'year': '%Y', 'year-month': '%Y-%m'} + # PeriodicSpider requires date parameters to be always set. + date_required = True + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -501,14 +504,6 @@ def __init__(self, *args, **kwargs): else: self.start_requests_callback = self.parse - @classmethod - def from_crawler(cls, crawler, from_date=None, *args, **kwargs): - # BaseSpider will only set `from_date` to its default value if `until_date` is set. - if not from_date: - from_date = cls.default_from_date - - return super().from_crawler(crawler, from_date=from_date, *args, **kwargs) - @classmethod def get_default_until_date(cls, spider): """ diff --git a/kingfisher_scrapy/spiders/paraguay_dncp_base.py b/kingfisher_scrapy/spiders/paraguay_dncp_base.py index 5bd534cc9..7e7bd69fd 100644 --- a/kingfisher_scrapy/spiders/paraguay_dncp_base.py +++ b/kingfisher_scrapy/spiders/paraguay_dncp_base.py @@ -25,6 +25,9 @@ class ParaguayDNCPBaseSpider(SimpleSpider): request_token = None max_attempts = 10 data_type = None + + # ParaguayDNCPBaseSpider requires date parameters to be always set. + date_required = True default_from_date = '2010-01-01T00:00:00' date_format = 'datetime' @@ -37,9 +40,6 @@ class ParaguayDNCPBaseSpider(SimpleSpider): @classmethod def from_crawler(cls, crawler, from_date=None, *args, **kwargs): - if not from_date: - from_date = cls.default_from_date - spider = super().from_crawler(crawler, from_date=from_date, *args, **kwargs) spider.request_token = crawler.settings.get('KINGFISHER_PARAGUAY_DNCP_REQUEST_TOKEN')