Skip to content

Commit

Permalink
spiders(kosovo): check for empty packages
Browse files Browse the repository at this point in the history
  • Loading branch information
yolile committed Nov 30, 2021
1 parent 06428e0 commit 456b350
Showing 1 changed file with 13 additions and 7 deletions.
20 changes: 13 additions & 7 deletions kingfisher_scrapy/spiders/kosovo.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from datetime import timedelta

from kingfisher_scrapy.util import parameters
from kingfisher_scrapy.util import parameters, handle_http_error
from kingfisher_scrapy.base_spider import SimpleSpider


Expand All @@ -27,13 +27,19 @@ class Kosovo(SimpleSpider):
data_type = 'release_package'

def start_requests(self):
# The API is slow even with short periods, so we request one day at a time
# The API is slow even with short periods, so we request one day at a time.
delta = self.until_date - self.from_date
for days in reversed(range(delta.days + 1)):
start = self.from_date + timedelta(days=days-1)
end = self.from_date + timedelta(days=days)
yield self.build_request(
'https://ocdskrpp.rks-gov.net/krppapi/tenderrelease?endDateFrom='
f'{start.strftime("%Y-%m-%d")}&endDateEnd={end.strftime("%Y-%m-%d")}',
formatter=parameters('endDateFrom', 'endDateEnd')
)
url = f'https://ocdskrpp.rks-gov.net/krppapi/tenderrelease?endDateFrom=' \
f'{start.strftime("%Y-%m-%d")}&endDateEnd={end.strftime("%Y-%m-%d")}'

yield self.build_request(url, formatter=parameters('endDateFrom', 'endDateEnd'))

@handle_http_error
def parse(self, response):
data = response.json()
# The API returns a release package with an empty releases array if no releases were found.
if data['releases']:
yield self.build_file_from_response(response, data_type=self.data_type)

0 comments on commit 456b350

Please sign in to comment.