Rename handle_error to handle_http_error

open-contracting · Jun 2, 2020 · 9375dc4 · 9375dc4
1 parent ce11307
commit 9375dc4
Show file tree

Hide file tree

Showing 34 changed files with 74 additions and 74 deletions.
diff --git a/docs/writing-spiders.rst b/docs/writing-spiders.rst
@@ -53,7 +53,7 @@ Here is a sample:
 .. code-block:: python
 
     from kingfisher_scrapy.base_spider import SimpleSpider
-    from kingfisher_scrapy.util import components, handle_error
+    from kingfisher_scrapy.util import components, handle_http_error
 
     class VerySimple(SimpleSpider):
         name = 'very_simple'

diff --git a/kingfisher_scrapy/base_spider.py b/kingfisher_scrapy/base_spider.py
@@ -10,7 +10,7 @@
 from kingfisher_scrapy import util
 from kingfisher_scrapy.exceptions import SpiderArgumentError
 from kingfisher_scrapy.items import File, FileError, FileItem
-from kingfisher_scrapy.util import handle_error
+from kingfisher_scrapy.util import handle_http_error
 
 
 class BaseSpider(scrapy.Spider):
@@ -257,7 +257,7 @@ def start_requests(self):
     encoding = 'utf-8'
     data_pointer = ''
 
-    @handle_error
+    @handle_http_error
     def parse(self, response):
         kwargs = {}
         if self.data_pointer:
@@ -305,7 +305,7 @@ def start_requests(self):
     encoding = 'utf-8'
     zip_file_format = None
 
-    @handle_error
+    @handle_http_error
     def parse(self, response):
         if self.zip_file_format:
             yield self.build_file_from_response(response, data_type='zip', post_to_api=False)
@@ -362,7 +362,7 @@ def start_requests(self):
 
     next_pointer = '/links/next'
 
-    @handle_error
+    @handle_http_error
     def parse(self, response):
         yield from super().parse(response)
 

diff --git a/kingfisher_scrapy/spiders/afghanistan_records.py b/kingfisher_scrapy/spiders/afghanistan_records.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class AfghanistanRecords(SimpleSpider):
@@ -17,7 +17,7 @@ def start_requests(self):
         url = 'https://ocds.ageops.net/api/ocds/records'
         yield scrapy.Request(url, meta={'kf_filename': 'list.json'}, callback=self.parse_list)
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         urls = json.loads(response.text)
         if self.sample:

diff --git a/kingfisher_scrapy/spiders/afghanistan_releases.py b/kingfisher_scrapy/spiders/afghanistan_releases.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class AfghanistanReleases(SimpleSpider):
@@ -17,7 +17,7 @@ def start_requests(self):
         url = 'https://ocds.ageops.net/api/ocds/releases/dates'
         yield scrapy.Request(url, meta={'kf_filename': 'list.json'}, callback=self.parse_list)
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         urls = json.loads(response.text)
         if self.sample:
@@ -27,7 +27,7 @@ def parse_list(self, response):
             # URL looks like https://ocds.ageops.net/api/ocds/releases/2020-05-30
             yield self.build_request(url, formatter=components(-1), callback=self.parse_release_list)
 
-    @handle_error
+    @handle_http_error
     def parse_release_list(self, response):
         urls = json.loads(response.text)
         if self.sample:

diff --git a/kingfisher_scrapy/spiders/argentina_buenos_aires.py b/kingfisher_scrapy/spiders/argentina_buenos_aires.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import ZipSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class ArgentinaBuenosAires(ZipSpider):
@@ -28,7 +28,7 @@ def start_requests(self):
         url = 'https://data.buenosaires.gob.ar/api/3/action/package_show?id=buenos-aires-compras'
         yield scrapy.Request(url, meta={'kf_filename': 'list.json'}, callback=self.parse_list)
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         data = json.loads(response.text)
         for resource in data['result']['resources']:

diff --git a/kingfisher_scrapy/spiders/australia_nsw.py b/kingfisher_scrapy/spiders/australia_nsw.py
@@ -1,7 +1,7 @@
 import json
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import handle_error, parameters
+from kingfisher_scrapy.util import handle_http_error, parameters
 
 
 class AustraliaNSW(SimpleSpider):
@@ -18,7 +18,7 @@ def start_requests(self):
                 callback=self.parse_list
             )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         data = json.loads(response.text)
         release_type = response.request.meta['release_type']

diff --git a/kingfisher_scrapy/spiders/canada_montreal.py b/kingfisher_scrapy/spiders/canada_montreal.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import handle_error, parameters, replace_parameter
+from kingfisher_scrapy.util import handle_http_error, parameters, replace_parameter
 
 
 class CanadaMontreal(SimpleSpider):
@@ -15,7 +15,7 @@ def start_requests(self):
         url = 'https://ville.montreal.qc.ca/vuesurlescontrats/api/releases.json?limit={step}'.format(step=self.step)
         yield scrapy.Request(url, meta={'kf_filename': 'offset-0.json'}, callback=self.parse_list)
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         yield from self.parse(response)
 

diff --git a/kingfisher_scrapy/spiders/chile_base.py b/kingfisher_scrapy/spiders/chile_base.py
@@ -2,7 +2,7 @@
 from datetime import date
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import components, date_range_by_month, handle_error
+from kingfisher_scrapy.util import components, date_range_by_month, handle_http_error
 
 
 class ChileCompraBaseSpider(SimpleSpider):
@@ -39,7 +39,7 @@ def start_requests(self):
                 callback=self.parse_list
             )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         data = json.loads(response.text)
         if 'status' in data and data['status'] != 200:

diff --git a/kingfisher_scrapy/spiders/colombia_bulk.py b/kingfisher_scrapy/spiders/colombia_bulk.py
@@ -1,7 +1,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import ZipSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class ColombiaBulk(ZipSpider):
@@ -29,7 +29,7 @@ def start_requests(self):
             callback=self.parse_list,
         )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         urls = response.xpath('//a[@class="enlaces_contenido"]/@href').getall()
         if self.sample:

diff --git a/kingfisher_scrapy/spiders/digiwhist_base.py b/kingfisher_scrapy/spiders/digiwhist_base.py
@@ -4,7 +4,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import BaseSpider
-from kingfisher_scrapy.util import handle_error
+from kingfisher_scrapy.util import handle_http_error
 
 
 class DigiwhistBase(BaseSpider):
@@ -13,7 +13,7 @@ def start_requests(self):
         for url in self.start_urls:
             yield scrapy.Request(url, dont_filter=True, meta={'kf_filename': 'file.tar.gz'})
 
-    @handle_error
+    @handle_http_error
     def parse(self, response):
         yield self.build_file_from_response(response, data_type='tar.gz', post_to_api=False)
 

diff --git a/kingfisher_scrapy/spiders/dominican_republic.py b/kingfisher_scrapy/spiders/dominican_republic.py
@@ -5,7 +5,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import BaseSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class DominicanRepublic(BaseSpider):
@@ -18,7 +18,7 @@ def start_requests(self):
             callback=self.parse_list,
         )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         urls = response.css('.fileLink::attr(href)').getall()
         json_urls = list(filter(lambda x: '/JSON_DGCP_' in x, urls))
@@ -30,7 +30,7 @@ def parse_list(self, response):
             if '/JSON_DGCP_' in url:
                 yield self.build_request('https:' + url, formatter=components(-1))
 
-    @handle_error
+    @handle_http_error
     def parse(self, response):
         file = tempfile.NamedTemporaryFile(delete=False)
         file.write(response.body)

diff --git a/kingfisher_scrapy/spiders/france.py b/kingfisher_scrapy/spiders/france.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import components, handle_error, parameters
+from kingfisher_scrapy.util import components, handle_http_error, parameters
 
 
 class France(SimpleSpider):
@@ -15,7 +15,7 @@ def start_requests(self):
         url = 'https://www.data.gouv.fr/api/1/datasets/?organization=534fff75a3a7292c64a77de4'
         yield scrapy.Request(url, meta={'kf_filename': 'page-1.json'}, callback=self.parse_list)
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         data = json.loads(response.text)
         for item in data['data']:

diff --git a/kingfisher_scrapy/spiders/honduras_oncae.py b/kingfisher_scrapy/spiders/honduras_oncae.py
@@ -1,7 +1,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import ZipSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class HondurasONCAE(ZipSpider):
@@ -18,7 +18,7 @@ def start_requests(self):
             callback=self.parse_list
         )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         urls = response.xpath('//a[contains(., "[json]")]/@href').getall()
         if self.sample:

diff --git a/kingfisher_scrapy/spiders/honduras_portal_bulk_files.py b/kingfisher_scrapy/spiders/honduras_portal_bulk_files.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class HondurasPortalBulkFiles(SimpleSpider):
@@ -17,7 +17,7 @@ def start_requests(self):
             callback=self.parse_list,
         )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         items = json.loads(response.text)
         if self.sample:

diff --git a/kingfisher_scrapy/spiders/indonesia_bandung.py b/kingfisher_scrapy/spiders/indonesia_bandung.py
@@ -2,7 +2,7 @@
 from datetime import date
 
 from kingfisher_scrapy.base_spider import BaseSpider
-from kingfisher_scrapy.util import components, date_range_by_year, handle_error, join, parameters
+from kingfisher_scrapy.util import components, date_range_by_year, handle_http_error, join, parameters
 
 
 class IndonesiaBandung(BaseSpider):
@@ -17,7 +17,7 @@ def start_requests(self):
         for year in date_range_by_year(start, stop):
             yield self.build_request(pattern.format(year), formatter=components(-1), callback=self.parse_list)
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         data = json.loads(response.text)
         for item in data['data']:
@@ -32,7 +32,7 @@ def parse_list(self, response):
                 yield self.build_request(next_page_url, formatter=join(components(-1), parameters('page')),
                                          callback=self.parse_list)
 
-    @handle_error
+    @handle_http_error
     def parse(self, response):
         data = json.loads(response.text)
         if len(data) == 0:

diff --git a/kingfisher_scrapy/spiders/kenya_makueni.py b/kingfisher_scrapy/spiders/kenya_makueni.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import handle_error, parameters
+from kingfisher_scrapy.util import handle_http_error, parameters
 
 
 class KenyaMakueni(SimpleSpider):
@@ -24,7 +24,7 @@ def start_requests(self):
                 callback=self.parse_count
             )
 
-    @handle_error
+    @handle_http_error
     def parse_count(self, response):
         total = int(response.text)
         for page in range(ceil(total / self.step)):

diff --git a/kingfisher_scrapy/spiders/malta.py b/kingfisher_scrapy/spiders/malta.py
@@ -4,7 +4,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import ZipSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class Malta(ZipSpider):
@@ -18,7 +18,7 @@ def start_requests(self):
             callback=self.parse_list
         )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         urls = json.loads(response.text)['packagesPerMonth']
         if self.sample:

diff --git a/kingfisher_scrapy/spiders/mexico_administracion_publica_federal.py b/kingfisher_scrapy/spiders/mexico_administracion_publica_federal.py
@@ -4,7 +4,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import handle_error, parameters, replace_parameter
+from kingfisher_scrapy.util import handle_http_error, parameters, replace_parameter
 
 
 class MexicoAdministracionPublicaFederal(SimpleSpider):
@@ -18,7 +18,7 @@ def start_requests(self):
         url = 'https://api.datos.gob.mx/v1/contratacionesabiertas'
         yield scrapy.Request(url, meta={'kf_filename': 'page-1.json'}, callback=self.parse_list)
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         yield from self.parse(response)
 

diff --git a/kingfisher_scrapy/spiders/mexico_cdmx.py b/kingfisher_scrapy/spiders/mexico_cdmx.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import components, handle_error
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class MexicoCDMXSource(SimpleSpider):
@@ -17,7 +17,7 @@ def start_requests(self):
             callback=self.parse_list
         )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         items = json.loads(response.text)
         if self.sample:

diff --git a/kingfisher_scrapy/spiders/mexico_inai.py b/kingfisher_scrapy/spiders/mexico_inai.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import components, handle_error, parameters
+from kingfisher_scrapy.util import components, handle_http_error, parameters
 
 
 class MexicoINAI(SimpleSpider):
@@ -19,7 +19,7 @@ def start_requests(self):
             callback=self.parse_list
         )
 
-    @handle_error
+    @handle_http_error
     def parse_list(self, response):
         datas = json.loads(response.text)
         for result in datas['result']['results']: