-
Notifications
You must be signed in to change notification settings - Fork 12
/
france.py
32 lines (26 loc) · 1.03 KB
/
france.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
import scrapy
from kingfisher_scrapy.base_spider import SimpleSpider
from kingfisher_scrapy.util import components, handle_http_error
class France(SimpleSpider):
"""
Domain
France
Swagger API documentation
https://doc.data.gouv.fr/api/reference/
"""
name = 'france'
# SimpleSpider
data_type = 'release_package'
def start_requests(self):
# A CKAN API JSON response.
# Ministère de l'économie, des finances et de la relance
url = 'https://www.data.gouv.fr/api/1/datasets/donnees-essentielles-de-la-commande-publique-fichiers' \
'-consolides/'
yield scrapy.Request(url, meta={'file_name': 'page-1.json'}, callback=self.parse_list)
@handle_http_error
def parse_list(self, response):
data = response.json()
for resource in data['resources']:
description = resource['description']
if description and 'ocds' in description.lower():
yield self.build_request(resource['url'], formatter=components(-2))