Skip to content

Commit

Permalink
Merge c934ed0 into 580065d
Browse files Browse the repository at this point in the history
  • Loading branch information
yolile committed Jun 29, 2020
2 parents 580065d + c934ed0 commit 13f7a71
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 3 deletions.
3 changes: 2 additions & 1 deletion kingfisher_scrapy/item_schema/item.json
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@
"release_package_json_lines",
"record_package_json_lines",
"release_package_in_ocdsReleasePackage_in_list_in_results",
"release_in_Release"
"release_in_Release",
"zip"
]
},
"encoding": {
Expand Down
5 changes: 5 additions & 0 deletions kingfisher_scrapy/spiders/chile_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ def start_requests(self):
@handle_http_error
def parse_list(self, response):
data = json.loads(response.text)
# some files contain invalid packages, eg:
# {
# "status": 500,
# "detail": "error"
# }
if 'status' in data and data['status'] != 200:
yield self.build_file_error_from_response(response, errors={'http_code': data['status']})
return
Expand Down
20 changes: 18 additions & 2 deletions kingfisher_scrapy/spiders/chile_compra_bulk.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import json
from datetime import date

from kingfisher_scrapy.base_spider import ZipSpider
from kingfisher_scrapy.items import FileError
from kingfisher_scrapy.util import components, date_range_by_month


Expand All @@ -10,7 +12,7 @@ class ChileCompraBulk(ZipSpider):
https://desarrolladores.mercadopublico.cl/OCDS/DescargaMasiva
Spider arguments
sample
Download only data released on February 2017.
Download only data released this month.
"""
name = 'chile_compra_bulk'
data_type = 'record_package'
Expand All @@ -27,6 +29,20 @@ def start_requests(self):
stop = date.today().replace(day=1)
if self.sample:
start = stop

for d in date_range_by_month(start, stop):
yield self.build_request(url.format(d), formatter=components(-1))

def build_file(self, file_name=None, url=None, data=None, data_type=None, encoding='utf-8', post_to_api=True):
json_data = json.loads(data)
# some files contain invalid record packages, eg:
# {
# "status": 500,
# "detail": "error"
# }
if 'status' in json_data and json_data['status'] != 200:
return FileError({
'url': url,
'errors': {'http_code': json_data['status']},
})
else:
return super().build_file(data=data, file_name=file_name, url=url, data_type=data_type, encoding=encoding)

0 comments on commit 13f7a71

Please sign in to comment.