Skip to content

Commit

Permalink
nigeria_budeshi: Share logic with Kaduna State Budeshi spiders
Browse files Browse the repository at this point in the history
  • Loading branch information
jpmckinney committed Apr 20, 2022
1 parent 34fc1ee commit a7b9bf5
Show file tree
Hide file tree
Showing 6 changed files with 22 additions and 58 deletions.
8 changes: 5 additions & 3 deletions kingfisher_scrapy/spiders/nigeria_budeshi_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,19 @@


class NigeriaBudeshiBase(SimpleSpider):
# NigeriaBudeshiBase
url_prefix = 'https://budeshi.ng/api/'

def start_requests(self):
yield scrapy.Request(
'https://budeshi.ng/api/project_list',
f'{self.url_prefix}project_list',
meta={'file_name': 'project_list.json'},
callback=self.parse_list
)

@handle_http_error
def parse_list(self, response):
project_list = response.json()
for project in sorted(project_list, key=itemgetter('year'), reverse=True):
for project in sorted(response.json(), key=itemgetter('year'), reverse=True):
yield from self.build_urls(project)

@abstractmethod
Expand Down
3 changes: 1 addition & 2 deletions kingfisher_scrapy/spiders/nigeria_budeshi_records.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,4 @@ class NigeriaBudeshiRecords(NigeriaBudeshiBase):
data_type = 'record_package'

def build_urls(self, project):
url = 'https://budeshi.ng/api/record/{}'
yield self.build_request(url.format(project['id']), formatter=components(-2))
yield self.build_request(f'{self.url_prefix}record/{project["id"]}', formatter=components(-2))
8 changes: 4 additions & 4 deletions kingfisher_scrapy/spiders/nigeria_budeshi_releases.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@ class NigeriaBudeshiReleases(NigeriaBudeshiBase):
name = 'nigeria_budeshi_releases'

# BaseSpider
skip_pluck = 'Already covered (see code for details)' # nigeria_budeshi_records
skip_pluck = 'Already covered (see code for details)' # nigeria*_budeshi_records

# SimpleSpider
data_type = 'release_package'

def build_urls(self, project):
url = 'https://budeshi.ng/api/releases/{id}/{tag}'
for tag in ('planning', 'tender', 'award', 'contract'):
yield self.build_request(url.format(id=project['id'], tag=tag), formatter=components(-2))
yield self.build_request(f'{self.url_prefix}releases/{project["id"]}/{tag}', formatter=components(-2))

def parse(self, response):
data = response.json()
# some responses include a release list with null objects, eg:
# Some responses include a release list with null objects, e.g.:
#
# "releases": [
# null
# ]
Expand Down
28 changes: 0 additions & 28 deletions kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_base.py

This file was deleted.

16 changes: 6 additions & 10 deletions kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_records.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,15 @@
from kingfisher_scrapy.spiders.nigeria_kaduna_state_budeshi_base import NigeriaKadunaStateBudeshiBase
from kingfisher_scrapy.spiders.nigeria_budeshi_records import NigeriaBudeshiRecords
from kingfisher_scrapy.util import components


class NigeriaKadunaStateBudeshiRecords(NigeriaKadunaStateBudeshiBase):
class NigeriaKadunaStateBudeshiRecords(NigeriaBudeshiRecords):
"""
Domain
Budeshi Nigeria - Kaduna State
Budeshi Nigeria - Kaduna State
API documentation
https://www.budeshi.ng/kadppa/api
https://www.budeshi.ng/kadppa/api
"""
name = 'nigeria_kaduna_state_budeshi_records'

# SimpleSpider
data_type = 'record_package'

def build_urls(self, id):
url = f'{self.base_url}record/{id}'
yield self.build_request(url, formatter=components(-2))
# NigeriaBudeshiBase
url_prefix = 'https://www.budeshi.ng/kadppa/api/'
17 changes: 6 additions & 11 deletions kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_releases.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,15 @@
from kingfisher_scrapy.spiders.nigeria_kaduna_state_budeshi_base import NigeriaKadunaStateBudeshiBase
from kingfisher_scrapy.spiders.nigeria_budeshi_releases import NigeriaBudeshiReleases
from kingfisher_scrapy.util import components


class NigeriaKadunaStateBudeshiReleases(NigeriaKadunaStateBudeshiBase):
class NigeriaKadunaStateBudeshiReleases(NigeriaBudeshiReleases):
"""
Domain
Budeshi Nigeria - Kaduna State
Budeshi Nigeria - Kaduna State
API documentation
https://www.budeshi.ng/kadppa/api
https://www.budeshi.ng/kadppa/api
"""
name = 'nigeria_kaduna_state_budeshi_releases'

# SimpleSpider
data_type = 'release_package'

def build_urls(self, id):
for tag in ('planning', 'tender', 'award', 'contract'):
url = f'{self.base_url}releases/{id}/{tag}'
yield self.build_request(url, formatter=components(-2))
# NigeriaBudeshiBase
url_prefix = 'https://www.budeshi.ng/kadppa/api/'

0 comments on commit a7b9bf5

Please sign in to comment.