diff --git a/kingfisher_scrapy/spiders/nigeria_budeshi_base.py b/kingfisher_scrapy/spiders/nigeria_budeshi_base.py index 2e4913152..137659769 100644 --- a/kingfisher_scrapy/spiders/nigeria_budeshi_base.py +++ b/kingfisher_scrapy/spiders/nigeria_budeshi_base.py @@ -8,17 +8,19 @@ class NigeriaBudeshiBase(SimpleSpider): + # NigeriaBudeshiBase + url_prefix = 'https://budeshi.ng/api/' + def start_requests(self): yield scrapy.Request( - 'https://budeshi.ng/api/project_list', + f'{self.url_prefix}project_list', meta={'file_name': 'project_list.json'}, callback=self.parse_list ) @handle_http_error def parse_list(self, response): - project_list = response.json() - for project in sorted(project_list, key=itemgetter('year'), reverse=True): + for project in sorted(response.json(), key=itemgetter('year'), reverse=True): yield from self.build_urls(project) @abstractmethod diff --git a/kingfisher_scrapy/spiders/nigeria_budeshi_records.py b/kingfisher_scrapy/spiders/nigeria_budeshi_records.py index 08805b31c..039310149 100644 --- a/kingfisher_scrapy/spiders/nigeria_budeshi_records.py +++ b/kingfisher_scrapy/spiders/nigeria_budeshi_records.py @@ -15,5 +15,4 @@ class NigeriaBudeshiRecords(NigeriaBudeshiBase): data_type = 'record_package' def build_urls(self, project): - url = 'https://budeshi.ng/api/record/{}' - yield self.build_request(url.format(project['id']), formatter=components(-2)) + yield self.build_request(f'{self.url_prefix}record/{project["id"]}', formatter=components(-2)) diff --git a/kingfisher_scrapy/spiders/nigeria_budeshi_releases.py b/kingfisher_scrapy/spiders/nigeria_budeshi_releases.py index 21bd85adb..b6c3ee42e 100644 --- a/kingfisher_scrapy/spiders/nigeria_budeshi_releases.py +++ b/kingfisher_scrapy/spiders/nigeria_budeshi_releases.py @@ -12,19 +12,19 @@ class NigeriaBudeshiReleases(NigeriaBudeshiBase): name = 'nigeria_budeshi_releases' # BaseSpider - skip_pluck = 'Already covered (see code for details)' # nigeria_budeshi_records + skip_pluck = 'Already covered (see code for details)' # nigeria*_budeshi_records # SimpleSpider data_type = 'release_package' def build_urls(self, project): - url = 'https://budeshi.ng/api/releases/{id}/{tag}' for tag in ('planning', 'tender', 'award', 'contract'): - yield self.build_request(url.format(id=project['id'], tag=tag), formatter=components(-2)) + yield self.build_request(f'{self.url_prefix}releases/{project["id"]}/{tag}', formatter=components(-2)) def parse(self, response): data = response.json() - # some responses include a release list with null objects, eg: + # Some responses include a release list with null objects, e.g.: + # # "releases": [ # null # ] diff --git a/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_base.py b/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_base.py deleted file mode 100644 index 6a5578405..000000000 --- a/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_base.py +++ /dev/null @@ -1,28 +0,0 @@ -from abc import abstractmethod -from operator import itemgetter - -import scrapy - -from kingfisher_scrapy.base_spiders import SimpleSpider -from kingfisher_scrapy.util import handle_http_error - - -class NigeriaKadunaStateBudeshiBase(SimpleSpider): - base_url = 'https://www.budeshi.ng/kadppa/api/' - - def start_requests(self): - yield scrapy.Request( - f'{self.base_url}project_list', - meta={'file_name': 'project_list.json'}, - callback=self.parse_list - ) - - @handle_http_error - def parse_list(self, response): - project_list = response.json() - for project in sorted(project_list, key=itemgetter('year'), reverse=True): - yield from self.build_urls(project['id']) - - @abstractmethod - def build_urls(self, project): - pass diff --git a/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_records.py b/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_records.py index fbc41ae53..eaaca82c6 100644 --- a/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_records.py +++ b/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_records.py @@ -1,19 +1,15 @@ -from kingfisher_scrapy.spiders.nigeria_kaduna_state_budeshi_base import NigeriaKadunaStateBudeshiBase +from kingfisher_scrapy.spiders.nigeria_budeshi_records import NigeriaBudeshiRecords from kingfisher_scrapy.util import components -class NigeriaKadunaStateBudeshiRecords(NigeriaKadunaStateBudeshiBase): +class NigeriaKadunaStateBudeshiRecords(NigeriaBudeshiRecords): """ Domain - Budeshi Nigeria - Kaduna State + Budeshi Nigeria - Kaduna State API documentation - https://www.budeshi.ng/kadppa/api + https://www.budeshi.ng/kadppa/api """ name = 'nigeria_kaduna_state_budeshi_records' - # SimpleSpider - data_type = 'record_package' - - def build_urls(self, id): - url = f'{self.base_url}record/{id}' - yield self.build_request(url, formatter=components(-2)) + # NigeriaBudeshiBase + url_prefix = 'https://www.budeshi.ng/kadppa/api/' diff --git a/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_releases.py b/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_releases.py index 6d4a81fb7..0a68729ad 100644 --- a/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_releases.py +++ b/kingfisher_scrapy/spiders/nigeria_kaduna_state_budeshi_releases.py @@ -1,20 +1,15 @@ -from kingfisher_scrapy.spiders.nigeria_kaduna_state_budeshi_base import NigeriaKadunaStateBudeshiBase +from kingfisher_scrapy.spiders.nigeria_budeshi_releases import NigeriaBudeshiReleases from kingfisher_scrapy.util import components -class NigeriaKadunaStateBudeshiReleases(NigeriaKadunaStateBudeshiBase): +class NigeriaKadunaStateBudeshiReleases(NigeriaBudeshiReleases): """ Domain - Budeshi Nigeria - Kaduna State + Budeshi Nigeria - Kaduna State API documentation - https://www.budeshi.ng/kadppa/api + https://www.budeshi.ng/kadppa/api """ name = 'nigeria_kaduna_state_budeshi_releases' - # SimpleSpider - data_type = 'release_package' - - def build_urls(self, id): - for tag in ('planning', 'tender', 'award', 'contract'): - url = f'{self.base_url}releases/{id}/{tag}' - yield self.build_request(url, formatter=components(-2)) + # NigeriaBudeshiBase + url_prefix = 'https://www.budeshi.ng/kadppa/api/'