-
Notifications
You must be signed in to change notification settings - Fork 12
/
indonesia_bandung.py
63 lines (56 loc) · 1.96 KB
/
indonesia_bandung.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
from kingfisher_scrapy.base_spider import PeriodicSpider
from kingfisher_scrapy.util import components, handle_http_error, join, parameters
class IndonesiaBandung(PeriodicSpider):
"""
Domain
Bandung Integrated Resource Management System (BIRMS)
Spider arguments
from_date
Download only releases from this date onward (YYYY format).
If ``from_date`` is not provided defaults to 2013.
until_date
Download only releases until this date (YYYY format).
If ``until_date`` is not provided defaults to current year.
API endpoints
Get contract list by year
Link
``https://birms.bandung.go.id/api/contracts/year/{year}?page={page}``
Parameters
year
contract year number
page
page number
Get contract detail by OCID
Link
``https://birms.bandung.go.id/api/newcontract/ocds-afzrfb-{source-database}-{year}-{uniqid}``
Parameters
source-database
(b)irms or (s)pse
year
contract year number
uniqid
release id
"""
name = 'indonesia_bandung'
# BaseSpider
date_format = 'year'
default_from_date = '2013'
# SimpleSpider
data_type = 'release'
# PeriodicSpider
pattern = 'https://birms.bandung.go.id/api/packages/year/{}'
start_requests_callback = 'parse_list'
@handle_http_error
def parse_list(self, response):
data = response.json()
for item in data['data']:
url = item['uri']
if url:
yield self.build_request(url, self.get_formatter())
else:
next_page_url = data.get('next_page_url')
if next_page_url:
yield self.build_request(next_page_url, formatter=join(self.get_formatter(), parameters('page')),
callback=self.parse_list)
def get_formatter(self):
return components(-1)