Skip to content

Commit

Permalink
Add from_date and until_date parameters to uruguay_base
Browse files Browse the repository at this point in the history
Signed-off-by: Yohanna Lisnichuk <yohanitalisnichuk@gmail.com>
  • Loading branch information
yolile committed Jul 13, 2020
1 parent 3264a3f commit 02ac4c0
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 5 deletions.
2 changes: 1 addition & 1 deletion kingfisher_scrapy/base_spider.py
Expand Up @@ -42,7 +42,7 @@ class BaseSpider(scrapy.Spider):

MAX_SAMPLE = 10
MAX_RELEASES_PER_PACKAGE = 100
VALID_DATE_FORMATS = {'date': '%Y-%m-%d', 'datetime': '%Y-%m-%dT%H:%M:%S'}
VALID_DATE_FORMATS = {'date': '%Y-%m-%d', 'datetime': '%Y-%m-%dT%H:%M:%S', 'month-year': '%m-%Y'}

def __init__(self, sample=None, note=None, from_date=None, until_date=None,
date_format='date', *args, **kwargs):
Expand Down
12 changes: 8 additions & 4 deletions kingfisher_scrapy/spiders/uruguay_base.py
@@ -1,17 +1,21 @@
from datetime import date
from datetime import date, datetime

from kingfisher_scrapy.base_spider import SimpleSpider
from kingfisher_scrapy.util import components, date_range_by_month


class UruguayBase(SimpleSpider):
download_delay = 0.9
default_from_date = '11-2017'

@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
return super(UruguayBase, cls).from_crawler(crawler, date_format='month-year', *args, **kwargs)

def start_requests(self):
url = 'http://comprasestatales.gub.uy/ocds/rss/{0.year:d}/{0.month:02d}'

start = date(2017, 11, 1)
stop = date.today().replace(day=1)
start = self.from_date if self.from_date else datetime.strptime(self.default_from_date, self.date_format)
stop = self.until_date if self.until_date else date.today().replace(day=1)
if self.sample:
start = stop

Expand Down

0 comments on commit 02ac4c0

Please sign in to comment.