Skip to content

Commit

Permalink
Add http error handler
Browse files Browse the repository at this point in the history
Signed-off-by: nativaldezt <nativaldezt@gmail.com>
  • Loading branch information
nativaldezt committed Dec 9, 2020
1 parent 51064ee commit e67a1e2
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions kingfisher_scrapy/spiders/moldova_positive_initiative.py
@@ -1,6 +1,6 @@
import scrapy
from kingfisher_scrapy.base_spider import SimpleSpider
from kingfisher_scrapy.util import components, get_parameter_value
from kingfisher_scrapy.util import components, get_parameter_value, handle_http_error


class MoldovaPositiveInitiative(SimpleSpider):
Expand All @@ -15,8 +15,10 @@ def start_requests(self):
url = 'https://www.tender.health/ocdsrelease'
yield scrapy.Request(url, meta={'file_name': 'page.html'}, callback=self.scrape_page)

@handle_http_error
def scrape_page(self, response, **kwargs):
for href in response.xpath('//a/@href').getall():
hrefs = response.xpath('//a/@href').getall()
for href in hrefs:
if '.json' in href:
url = get_parameter_value(href, 'q')
yield self.build_request(url, formatter=components(-1))

0 comments on commit e67a1e2

Please sign in to comment.