Merge c6755df into e3140c2

open-contracting · Nov 12, 2020 · 4031d00 · 4031d00
2 parents e3140c2 + c6755df
commit 4031d00
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 17 deletions.
diff --git a/kingfisher_scrapy/spiders/bolivia_agetic.py b/kingfisher_scrapy/spiders/bolivia_agetic.py
@@ -3,7 +3,7 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import handle_http_error, components
+from kingfisher_scrapy.util import components, handle_http_error
 
 
 class BoliviaAgetic(SimpleSpider):

diff --git a/kingfisher_scrapy/spiders/nigeria_portal.py b/kingfisher_scrapy/spiders/nigeria_portal.py
@@ -1,3 +1,5 @@
+import re
+
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider, browser_user_agent
@@ -18,22 +20,38 @@ class NigeriaPortal(SimpleSpider):
     def start_requests(self):
         yield scrapy.Request(
             'http://nocopo.bpp.gov.ng/OpenData.aspx',
-            meta={'file_name': 'form.html'},
-            callback=self.parse_list
+            meta={'file_name': 'page-0.html', 'page': 0},
+            callback=self.parse_list,
         )
 
     @handle_http_error
     def parse_list(self, response):
-        formdata = {
-            '__VIEWSTATE': response.css('input#__VIEWSTATE::attr(value)').extract_first(),
-            '__VIEWSTATEGENERATOR': 'CA0B0334',
-            '__EVENTVALIDATION': response.css('input#__EVENTVALIDATION::attr(value)').extract_first(),
-            'dnn$ctr561$no_JsonReport$lbtnExportAll': 'Export Checked to JSON',
-        }
-
-        checks = response.css('input').xpath('@name').getall()
-        for item in checks:
-            if 'dnn$ctr' in item and 'chbIsDoing' in item:
+        self.logger.debug('Crawled page {}'.format(response.xpath('//td[@colspan="8"]/span/text()').get()))
+
+        page = response.request.meta['page']
+
+        formdata = {}
+        for item in response.xpath('//input/@name').getall():
+            if re.search(r'^dnn\$ctr561\$no_JsonReport\$DGno_Proc_PlanningPublished\$ctl\d+\$chbIsDoing$', item):
                 formdata.update({item: 'on'})
 
-        yield scrapy.FormRequest.from_response(response, formdata=formdata, meta={'file_name': 'all.json'})
+        yield scrapy.FormRequest.from_response(
+            response,
+            formdata=formdata,
+            clickdata={'name': 'dnn$ctr561$no_JsonReport$lbtnExportAll'},
+            meta={'file_name': f'page-{page}.json'},
+        )
+
+        # Get the next pagination link, and simulate clicking on it.
+        href = response.xpath('//td[@colspan="8"]/span/following-sibling::a[1]/@href').get()
+        if href:
+            yield scrapy.FormRequest.from_response(
+                response,
+                formdata={
+                    '__EVENTTARGET': re.search(r"'(.+?)'", href)[1],
+                    '__EVENTARGUMENT': '',
+                },
+                dont_click=True,
+                meta={'file_name': f'page-{page}.html', 'page': page + 1},
+                callback=self.parse_list,
+            )
diff --git a/tests/pipelines/test_unflatten.py b/tests/pipelines/test_unflatten.py
@@ -1,8 +1,7 @@
-from openpyxl import Workbook
-from openpyxl.writer.excel import save_virtual_workbook
-
 import pytest
 from flattentool.input import BadXLSXZipFile
+from openpyxl import Workbook
+from openpyxl.writer.excel import save_virtual_workbook
 
 from kingfisher_scrapy.items import File
 from kingfisher_scrapy.pipelines import Unflatten