Update date format to allow just date or datetime

Signed-off-by: Yohanna Lisnichuk <yohanitalisnichuk@gmail.com>
open-contracting · May 13, 2020 · 879a52e · 879a52e
1 parent 5458901
commit 879a52e
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 7 deletions.
diff --git a/kingfisher_scrapy/base_spider.py b/kingfisher_scrapy/base_spider.py
@@ -41,17 +41,18 @@ class BaseSpider(scrapy.Spider):
 
     MAX_SAMPLE = 10
     MAX_RELEASES_PER_PACKAGE = 100
+    VALID_DATE_FORMATS = {'year_month_day': '%Y-%m-%d', 'year_month_day_time': '%Y-%m-%dT%H:%M:%S'}
 
-    def __init__(self, sample=None, note=None, from_date=None, until_date=None, date_format='%Y-%m-%d',
-                 *args, **kwargs):
+    def __init__(self, sample=None, note=None, from_date=None, until_date=None,
+                 date_format='year_month_day', *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         # https://docs.scrapy.org/en/latest/topics/spiders.html#spider-arguments
         self.sample = sample == 'true'
         self.from_date = from_date
         self.until_date = until_date
         self.note = note
-        self.date_format = date_format
+        self.date_format = self.VALID_DATE_FORMATS[date_format]
 
         spider_arguments = {
             'sample': sample,
@@ -74,7 +75,6 @@ def from_crawler(cls, crawler, *args, **kwargs):
             if not spider.until_date:
                 # 'until_date' defaults to today
                 spider.until_date = datetime.now().strftime(spider.date_format)
-
             try:
                 spider.from_date = datetime.strptime(spider.from_date, spider.date_format)
             except ValueError as e:

diff --git a/kingfisher_scrapy/spiders/paraguay_dncp_base.py b/kingfisher_scrapy/spiders/paraguay_dncp_base.py
@@ -37,7 +37,7 @@ class ParaguayDNCPBaseSpider(BaseSpider):
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
-        spider = super(ParaguayDNCPBaseSpider, cls).from_crawler(crawler, date_format='%Y-%m-%dT%H:%M:%S',
+        spider = super(ParaguayDNCPBaseSpider, cls).from_crawler(crawler, date_format='year_month_day_time',
                                                                  *args, **kwargs)
 
         spider.request_token = crawler.settings.get('KINGFISHER_PARAGUAY_DNCP_REQUEST_TOKEN')
@@ -50,9 +50,8 @@ def from_crawler(cls, crawler, *args, **kwargs):
 
     def start_requests(self):
         if self.from_date:
-            self.from_date = self.from_date.strftime(self.date_format)
             self.base_page_url = '{}/search/processes?tipo_fecha=fecha_release&fecha_desde={}'\
-                .format(self.base_url, self.from_date)
+                .format(self.base_url, self.from_date.strftime(self.date_format))
         yield scrapy.Request(
             self.base_page_url,
             # send duplicate requests when the token expired and in the continuation of last_request saved.