Skip to content

Commit

Permalink
chore(united_kingdom_*): Simplify default until_date
Browse files Browse the repository at this point in the history
  • Loading branch information
jpmckinney committed Sep 20, 2023
1 parent db7d544 commit 5c21481
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 3 deletions.
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from datetime import datetime

import scrapy

from kingfisher_scrapy.base_spiders import LinksSpider
Expand Down Expand Up @@ -32,8 +34,7 @@ def start_requests(self):
until_date = self.until_date.strftime(self.date_format)
url = f'{url}&publishedFrom={from_date}&publishedTo={until_date}'
else:
until_date = self.get_default_until_date(self).strftime(self.date_format)

until_date = datetime.utcnow().strftime(self.date_format)
yield scrapy.Request(url, meta={'file_name': f'{until_date}.json'}) # reverse chronological order

@handle_http_error
Expand Down
4 changes: 3 additions & 1 deletion kingfisher_scrapy/spiders/united_kingdom_fts.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from datetime import datetime

import scrapy

from kingfisher_scrapy.base_spiders import LinksSpider
Expand Down Expand Up @@ -37,7 +39,7 @@ def start_requests(self):
until_date = self.until_date.strftime(self.date_format)
url = f'{url}?updatedFrom={from_date}&updatedTo={until_date}'
else:
until_date = self.get_default_until_date(self).strftime(self.date_format)
until_date = datetime.utcnow().strftime(self.date_format)
yield scrapy.Request(url, meta={'file_name': f'{until_date}.json'}, # reverse chronological order
headers={'Accept': 'application/json'})

Expand Down

0 comments on commit 5c21481

Please sign in to comment.