spiders(united_kingdom_contracts_finder): download record packages in…

…stead of release packages
open-contracting · Apr 5, 2022 · 851831f · 851831f
1 parent 37fff6b
commit 851831f
Show file tree

Hide file tree

Showing 6 changed files with 286 additions and 3 deletions.
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/kingfisher_scrapy/spiders/united_kingdom_contracts_finder.py b/kingfisher_scrapy/spiders/united_kingdom_contracts_finder.py
@@ -14,10 +14,9 @@ class UnitedKingdomContractsFinder(IndexSpider):
 
     # BaseSpider
     ocds_version = '1.0'  # uses deprecated fields
-    root_path = 'results.item'
 
     # SimpleSpider
-    data_type = 'release_package'
+    data_type = 'record_package'
 
     # IndexSpider
     total_pages_pointer = '/maxPage'
@@ -30,7 +29,11 @@ def start_requests(self):
 
     def parse(self, response, **kwargs):
         if self.is_http_success(response):
-            yield from super().parse(response)
+            for result in response.json()['results']:
+                for release in result['releases']:
+                    ocid = release["ocid"]
+                    url = f'https://www.contractsfinder.service.gov.uk/Published/OCDS/Record/{ocid}'
+                    yield scrapy.Request(url, meta={'file_name': f'{ocid}.json'}, callback=super().parse)
         else:
             request = response.request.copy()
             wait_time = int(response.headers.get('Retry-After', 1))