Skip to content

Commit

Permalink
add nigeria gombe state spider
Browse files Browse the repository at this point in the history
  • Loading branch information
Ravf95 committed Sep 30, 2021
1 parent d090af8 commit 6bf942e
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 0 deletions.
7 changes: 7 additions & 0 deletions docs/spiders.rst
Expand Up @@ -951,6 +951,13 @@ Nigeria
scrapy crawl nigeria_edo_state
.. autoclass:: kingfisher_scrapy.spiders.nigeria_gombe_state.NigeriaGombeState
:no-members:

.. code-block:: bash
scrapy crawl nigeria_gombe_state
.. autoclass:: kingfisher_scrapy.spiders.nigeria_kaduna_state_records.NigeriaKadunaStateRecords
:no-members:

Expand Down
28 changes: 28 additions & 0 deletions kingfisher_scrapy/spiders/nigeria_gombe_state.py
@@ -0,0 +1,28 @@
import scrapy

from kingfisher_scrapy.base_spider import SimpleSpider, browser_user_agent
from kingfisher_scrapy.util import handle_http_error, parameters


class NigeriaGombeState(SimpleSpider):
"""
Domain
Nigeria Gombe State Open Contracting Portal
"""
name = 'nigeria_gombe_state'
user_agent = browser_user_agent
base_url = 'http://gombe.stateopencontracting.com/{}'

# SimpleSpider
data_type = 'release_package'

def start_requests(self):
url = self.base_url.format('Other-Basic/Report/Json-Report')
yield scrapy.Request(url, meta={'file_name': 'page-0.html', 'page': 0}, callback=self.parse_list)

@handle_http_error
def parse_list(self, response):
pattern = '//table[@id="dnn_ctr561_no_JsonReport_DGno_Proc_PlanningPublished"]'\
'//tr[position()>1]/td[position()=1 and string-length(text())>1]/text()'
for item in response.xpath(pattern).getall():
yield self.build_request(self.base_url.format(f'ocdsjson.ashx?ocid={item}'), formatter=parameters('ocid'))

0 comments on commit 6bf942e

Please sign in to comment.