Merge 1a357d1 into f4e6d7b

kingfisher_scrapy/base_spider.py

@@ -72,7 +72,7 @@ class BaseSpider(scrapy.Spider):
     date_format = 'date'
 
     # Set `date_required` to True in class attribute to always set the `from` and `until` date parameters.
-    # If `date_required` is true the attribute `default_from_date` should be set too.
+    # If `date_required` is True, the attribute `default_from_date` should be set too.
     date_required = False
 
     def __init__(self, sample=None, note=None, from_date=None, until_date=None, crawl_time=None,
@@ -493,6 +493,9 @@ class PeriodicSpider(SimpleSpider):
     """
     VALID_DATE_FORMATS = {'year': '%Y', 'year-month': '%Y-%m'}
 
+    # PeriodicSpider requires date parameters to be always set.
+    date_required = True
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -501,14 +504,6 @@ def __init__(self, *args, **kwargs):
         else:
             self.start_requests_callback = self.parse
 
-    @classmethod
-    def from_crawler(cls, crawler, from_date=None, *args, **kwargs):
-        # BaseSpider will only set `from_date` to its default value if `until_date` is set.
-        if not from_date:
-            from_date = cls.default_from_date
-
-        return super().from_crawler(crawler, from_date=from_date, *args, **kwargs)
-
     @classmethod
     def get_default_until_date(cls, spider):
         """

kingfisher_scrapy/spiders/paraguay_dncp_base.py

@@ -25,6 +25,9 @@ class ParaguayDNCPBaseSpider(SimpleSpider):
     request_token = None
     max_attempts = 10
     data_type = None
+
+    # ParaguayDNCPBaseSpider requires date parameters to be always set.
+    date_required = True
     default_from_date = '2010-01-01T00:00:00'
     date_format = 'datetime'
 
@@ -37,9 +40,6 @@ class ParaguayDNCPBaseSpider(SimpleSpider):
 
     @classmethod
     def from_crawler(cls, crawler, from_date=None, *args, **kwargs):
-        if not from_date:
-            from_date = cls.default_from_date
-
         spider = super().from_crawler(crawler, from_date=from_date, *args, **kwargs)
 
         spider.request_token = crawler.settings.get('KINGFISHER_PARAGUAY_DNCP_REQUEST_TOKEN')