Skip to content

Commit

Permalink
fix test base spider and datastore extension date transform
Browse files Browse the repository at this point in the history
Signed-off-by: Yohanna Lisnichuk <yohanitalisnichuk@gmail.com>
  • Loading branch information
yolile committed May 26, 2021
1 parent 605b127 commit 827c4db
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 3 deletions.
3 changes: 1 addition & 2 deletions kingfisher_scrapy/extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,10 +198,9 @@ def spider_opened(self, spider):
default_from_date = None
if not spider.from_date or (spider.from_date == default_from_date):
self.logger.info('Getting the date from which to resume the crawl (if any)')
self.execute("SELECT max(data->>'date') FROM {table}", table=spider.name)
self.execute("SELECT max(data->>'date')::timestamptz FROM {table}", table=spider.name)
from_date = self.cursor.fetchone()[0]
if from_date:
from_date = datetime.strptime(from_date, '%Y-%m-%dT%H:%M:%S%z')
spider.from_date = datetime.strptime(datetime.strftime(from_date, spider.date_format),
spider.date_format)
self.connection.commit()
Expand Down
2 changes: 1 addition & 1 deletion tests/test_base_spider.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def test_qs_parameters(kwargs, expected):


def test_data_base_url_without_crawl_time():
expected = 'When using `DATABASE_URL` the spider argument `crawl_time` must be set'
expected = "spider argument `compile_releases`: can't be set if spider returns records"

with pytest.raises(SpiderArgumentError) as e:
spider_with_crawler(settings={'DATABASE_URL': 'test'})
Expand Down

0 comments on commit 827c4db

Please sign in to comment.