Merge pull request #1076 from open-contracting/digiwhist

feat: restore digiwhist and add seriba and north macedonia
open-contracting · Apr 12, 2024 · 2fd59fe · 2fd59fe
2 parents cc713a9 + a49f23d
commit 2fd59fe
Show file tree

Hide file tree

Showing 38 changed files with 77 additions and 56 deletions.
diff --git a/docs/spiders.rst b/docs/spiders.rst
@@ -976,6 +976,16 @@ Nigeria
 
    scrapy crawl nigeria_portal
 
+North Macedonia
+~~~~~~~~~~~~~~~
+
+.. autoclass:: kingfisher_scrapy.spiders.north_macedonia_digiwhist.NorthMacedoniaDigiwhist
+   :no-members:
+
+.. code-block:: bash
+
+   scrapy crawl north_macedonia_digiwhist
+
 Norway
 ~~~~~~
 
@@ -1136,6 +1146,16 @@ Romania
 
    scrapy crawl romania_digiwhist
 
+Serbia
+~~~~~~
+
+.. autoclass:: kingfisher_scrapy.spiders.serbia_digiwhist.SerbiaDigiwhist
+   :no-members:
+
+.. code-block:: bash
+
+   scrapy crawl serbia_digiwhist
+
 Slovakia
 ~~~~~~~~
 

diff --git a/kingfisher_scrapy/commands/updatedocs.py b/kingfisher_scrapy/commands/updatedocs.py
@@ -17,8 +17,8 @@ def run(self, args, opts):
 
         def _keyfunc(module):
             module_name = module.__name__.rsplit('.', 1)[-1]
-            if module_name.startswith(('costa_rica', 'czech_republic', 'dominican_republic', 'south_africa',
-                                       'united_kingdom')):
+            if module_name.startswith(('costa_rica', 'czech_republic', 'dominican_republic', 'north_macedonia',
+                                       'south_africa', 'united_kingdom')):
                 return '_'.join(module_name.split('_', 2)[:2])
             return module_name.split('_', 1)[0]
 

diff --git a/kingfisher_scrapy/spiders/austria_digiwhist.py b/kingfisher_scrapy/spiders/austria_digiwhist.py
@@ -3,4 +3,4 @@
 
 class AustriaDigiwhist(DigiwhistBase):
     name = 'austria_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/AT_ocds_data.json.tar.gz']
+    country_code = 'at'
diff --git a/kingfisher_scrapy/spiders/belgium_digiwhist.py b/kingfisher_scrapy/spiders/belgium_digiwhist.py
@@ -3,4 +3,4 @@
 
 class BelgiumDigiwhist(DigiwhistBase):
     name = 'belgium_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/BE_ocds_data.json.tar.gz']
+    country_code = 'be'
diff --git a/kingfisher_scrapy/spiders/bulgaria_digiwhist.py b/kingfisher_scrapy/spiders/bulgaria_digiwhist.py
@@ -3,4 +3,4 @@
 
 class BulgariaDigiwhist(DigiwhistBase):
     name = 'bulgaria_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/BG_ocds_data.json.tar.gz']
+    country_code = 'bg'
diff --git a/kingfisher_scrapy/spiders/croatia_digiwhist.py b/kingfisher_scrapy/spiders/croatia_digiwhist.py
@@ -3,4 +3,4 @@
 
 class CroatiaDigiwhist(DigiwhistBase):
     name = 'croatia_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/HR_ocds_data.json.tar.gz']
+    country_code = 'hr'
diff --git a/kingfisher_scrapy/spiders/cyprus_digiwhist.py b/kingfisher_scrapy/spiders/cyprus_digiwhist.py
@@ -3,4 +3,4 @@
 
 class CyprusDigiwhist(DigiwhistBase):
     name = 'cyprus_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/CY_ocds_data.json.tar.gz']
+    country_code = 'cy'
diff --git a/kingfisher_scrapy/spiders/czech_republic_digiwhist.py b/kingfisher_scrapy/spiders/czech_republic_digiwhist.py
@@ -3,4 +3,4 @@
 
 class CzechRepublicDigiwhist(DigiwhistBase):
     name = 'czech_republic_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/CZ_ocds_data.json.tar.gz']
+    country_code = 'cz'
diff --git a/kingfisher_scrapy/spiders/denmark_digiwhist.py b/kingfisher_scrapy/spiders/denmark_digiwhist.py
@@ -3,4 +3,4 @@
 
 class DenmarkDigiwhist(DigiwhistBase):
     name = 'denmark_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/DK_ocds_data.json.tar.gz']
+    country_code = 'dk'
diff --git a/kingfisher_scrapy/spiders/digiwhist_base.py b/kingfisher_scrapy/spiders/digiwhist_base.py
@@ -1,13 +1,8 @@
-import tarfile
-from io import BytesIO
+from kingfisher_scrapy.base_spiders import CompressedFileSpider
+from kingfisher_scrapy.util import browser_user_agent, components
 
-import scrapy
 
-from kingfisher_scrapy.base_spiders import BaseSpider
-from kingfisher_scrapy.util import browser_user_agent, handle_http_error
-
-
-class DigiwhistBase(BaseSpider):
+class DigiwhistBase(CompressedFileSpider):
     """
     Domain
       Digiwhist
@@ -19,19 +14,13 @@ class DigiwhistBase(BaseSpider):
     # BaseSpider
     line_delimited = True
 
-    # start_urls must be provided by subclasses.
+    # CompressedFileSpider
+    data_type = 'release_package'
 
-    def start_requests(self):
-        # See scrapy.spiders.Spider.start_requests
-        for url in self.start_urls:
-            yield scrapy.Request(url, meta={'file_name': 'file.tar.gz'})
+    # Local
+    base_url = 'https://opentender.eu/data/downloads/data-{}-json-json.zip'
 
-    @handle_http_error
-    def parse(self, response):
+    # country_code must be provided by subclasses.
 
-        # Load a line at the time, pass it to API
-        with tarfile.open(fileobj=BytesIO(response.body), mode="r:gz") as tar:
-            with tar.extractfile(tar.next()) as f:
-                yield self.build_file_from_response(
-                    response, data_type='release_package', file_name='data.json', data=f
-                )
+    def start_requests(self):
+        yield self.build_request(self.base_url.format(self.country_code), formatter=components(-1))
diff --git a/kingfisher_scrapy/spiders/estonia_digiwhist.py b/kingfisher_scrapy/spiders/estonia_digiwhist.py
@@ -3,4 +3,4 @@
 
 class EstoniaDigiwhist(DigiwhistBase):
     name = 'estonia_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/EE_ocds_data.json.tar.gz']
+    country_code = 'ee'
diff --git a/kingfisher_scrapy/spiders/europe_ted_digiwhist.py b/kingfisher_scrapy/spiders/europe_ted_digiwhist.py
@@ -3,4 +3,4 @@
 
 class EuropeTEDDigiwhist(DigiwhistBase):
     name = 'europe_ted_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/TED_ocds_data.json.tar.gz']
+    country_code = 'eu'
diff --git a/kingfisher_scrapy/spiders/finland_digiwhist.py b/kingfisher_scrapy/spiders/finland_digiwhist.py
@@ -3,4 +3,4 @@
 
 class FinlandDigiwhist(DigiwhistBase):
     name = 'finland_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/FI_ocds_data.json.tar.gz']
+    country_code = 'fi'
diff --git a/kingfisher_scrapy/spiders/france_digiwhist.py b/kingfisher_scrapy/spiders/france_digiwhist.py
@@ -3,4 +3,4 @@
 
 class FranceDigiwhist(DigiwhistBase):
     name = 'france_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/FR_ocds_data.json.tar.gz']
+    country_code = 'fr'
diff --git a/kingfisher_scrapy/spiders/georgia_digiwhist.py b/kingfisher_scrapy/spiders/georgia_digiwhist.py
@@ -3,4 +3,4 @@
 
 class GeorgiaDigiwhist(DigiwhistBase):
     name = 'georgia_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/GE_ocds_data.json.tar.gz']
+    country_code = 'ge'
diff --git a/kingfisher_scrapy/spiders/germany_digiwhist.py b/kingfisher_scrapy/spiders/germany_digiwhist.py
@@ -3,4 +3,4 @@
 
 class GermanyDigiwhist(DigiwhistBase):
     name = 'germany_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/DE_ocds_data.json.tar.gz']
+    country_code = 'de'
diff --git a/kingfisher_scrapy/spiders/greece_digiwhist.py b/kingfisher_scrapy/spiders/greece_digiwhist.py
@@ -3,4 +3,4 @@
 
 class GreeceDigiwhist(DigiwhistBase):
     name = 'greece_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/GR_ocds_data.json.tar.gz']
+    country_code = 'gr'
diff --git a/kingfisher_scrapy/spiders/hungary_digiwhist.py b/kingfisher_scrapy/spiders/hungary_digiwhist.py
@@ -3,4 +3,4 @@
 
 class HungaryDigiwhist(DigiwhistBase):
     name = 'hungary_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/HU_ocds_data.json.tar.gz']
+    country_code = 'hu'
diff --git a/kingfisher_scrapy/spiders/iceland_digiwhist.py b/kingfisher_scrapy/spiders/iceland_digiwhist.py
@@ -3,4 +3,4 @@
 
 class IcelandDigiwhist(DigiwhistBase):
     name = 'iceland_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/IS_ocds_data.json.tar.gz']
+    country_code = 'is'
diff --git a/kingfisher_scrapy/spiders/ireland_digiwhist.py b/kingfisher_scrapy/spiders/ireland_digiwhist.py
@@ -3,4 +3,4 @@
 
 class IrelandDigiwhist(DigiwhistBase):
     name = 'ireland_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/IE_ocds_data.json.tar.gz']
+    country_code = 'ie'
diff --git a/kingfisher_scrapy/spiders/italy_digiwhist.py b/kingfisher_scrapy/spiders/italy_digiwhist.py
@@ -3,4 +3,4 @@
 
 class ItalyDigiwhist(DigiwhistBase):
     name = 'italy_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/IT_ocds_data.json.tar.gz']
+    country_code = 'it'
diff --git a/kingfisher_scrapy/spiders/latvia_digiwhist.py b/kingfisher_scrapy/spiders/latvia_digiwhist.py
@@ -3,4 +3,4 @@
 
 class LatviaDigiwhist(DigiwhistBase):
     name = 'latvia_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/LV_ocds_data.json.tar.gz']
+    country_code = 'lv'
diff --git a/kingfisher_scrapy/spiders/lithuania_digiwhist.py b/kingfisher_scrapy/spiders/lithuania_digiwhist.py
@@ -3,4 +3,4 @@
 
 class LithuaniaDigiwhist(DigiwhistBase):
     name = 'lithuania_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/LT_ocds_data.json.tar.gz']
+    country_code = 'lt'
diff --git a/kingfisher_scrapy/spiders/luxembourg_digiwhist.py b/kingfisher_scrapy/spiders/luxembourg_digiwhist.py
@@ -3,4 +3,4 @@
 
 class LuxembourgDigiwhist(DigiwhistBase):
     name = 'luxembourg_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/LU_ocds_data.json.tar.gz']
+    country_code = 'lu'
diff --git a/kingfisher_scrapy/spiders/malta_digiwhist.py b/kingfisher_scrapy/spiders/malta_digiwhist.py
@@ -3,4 +3,4 @@
 
 class MaltaDigiwhist(DigiwhistBase):
     name = 'malta_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/MT_ocds_data.json.tar.gz']
+    country_code = 'mt'
diff --git a/kingfisher_scrapy/spiders/netherlands_digiwhist.py b/kingfisher_scrapy/spiders/netherlands_digiwhist.py
@@ -3,4 +3,4 @@
 
 class NetherlandsDigiwhist(DigiwhistBase):
     name = 'netherlands_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/NL_ocds_data.json.tar.gz']
+    country_code = 'nl'
diff --git a/kingfisher_scrapy/spiders/north_macedonia_digiwhist.py b/kingfisher_scrapy/spiders/north_macedonia_digiwhist.py
@@ -0,0 +1,6 @@
+from kingfisher_scrapy.spiders.digiwhist_base import DigiwhistBase
+
+
+class NorthMacedoniaDigiwhist(DigiwhistBase):
+    name = 'north_macedonia_digiwhist'
+    country_code = 'mk'
diff --git a/kingfisher_scrapy/spiders/norway_digiwhist.py b/kingfisher_scrapy/spiders/norway_digiwhist.py
@@ -3,4 +3,4 @@
 
 class NorwayDigiwhist(DigiwhistBase):
     name = 'norway_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/NO_ocds_data.json.tar.gz']
+    country_code = 'no'
diff --git a/kingfisher_scrapy/spiders/poland_digiwhist.py b/kingfisher_scrapy/spiders/poland_digiwhist.py
@@ -3,4 +3,4 @@
 
 class PolandDigiwhist(DigiwhistBase):
     name = 'poland_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/PL_ocds_data.json.tar.gz']
+    country_code = 'pl'
diff --git a/kingfisher_scrapy/spiders/portugal_digiwhist.py b/kingfisher_scrapy/spiders/portugal_digiwhist.py
@@ -3,4 +3,4 @@
 
 class PortugalDigiwhist(DigiwhistBase):
     name = 'portugal_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/PT_ocds_data.json.tar.gz']
+    country_code = 'pt'
diff --git a/kingfisher_scrapy/spiders/romania_digiwhist.py b/kingfisher_scrapy/spiders/romania_digiwhist.py
@@ -3,4 +3,4 @@
 
 class RomaniaDigiwhist(DigiwhistBase):
     name = 'romania_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/RO_ocds_data.json.tar.gz']
+    country_code = 'ro'
diff --git a/kingfisher_scrapy/spiders/serbia_digiwhist.py b/kingfisher_scrapy/spiders/serbia_digiwhist.py
@@ -0,0 +1,6 @@
+from kingfisher_scrapy.spiders.digiwhist_base import DigiwhistBase
+
+
+class SerbiaDigiwhist(DigiwhistBase):
+    name = 'serbia_digiwhist'
+    country_code = 'rs'
diff --git a/kingfisher_scrapy/spiders/slovakia_digiwhist.py b/kingfisher_scrapy/spiders/slovakia_digiwhist.py
@@ -3,4 +3,4 @@
 
 class SlovakiaDigiwhist(DigiwhistBase):
     name = 'slovakia_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/SK_ocds_data.json.tar.gz']
+    country_code = 'sk'
diff --git a/kingfisher_scrapy/spiders/slovenia_digiwhist.py b/kingfisher_scrapy/spiders/slovenia_digiwhist.py
@@ -3,4 +3,4 @@
 
 class SloveniaDigiwhist(DigiwhistBase):
     name = 'slovenia_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/SI_ocds_data.json.tar.gz']
+    country_code = 'si'
diff --git a/kingfisher_scrapy/spiders/spain_digiwhist.py b/kingfisher_scrapy/spiders/spain_digiwhist.py
@@ -3,4 +3,4 @@
 
 class SpainDigiwhist(DigiwhistBase):
     name = 'spain_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/ES_ocds_data.json.tar.gz']
+    country_code = 'es'
diff --git a/kingfisher_scrapy/spiders/sweden_digiwhist.py b/kingfisher_scrapy/spiders/sweden_digiwhist.py
@@ -3,4 +3,4 @@
 
 class SwedenDigiwhist(DigiwhistBase):
     name = 'sweden_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/SE_ocds_data.json.tar.gz']
+    country_code = 'se'
diff --git a/kingfisher_scrapy/spiders/switzerland_digiwhist.py b/kingfisher_scrapy/spiders/switzerland_digiwhist.py
@@ -3,4 +3,4 @@
 
 class SwitzerlandDigiwhist(DigiwhistBase):
     name = 'switzerland_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/CH_ocds_data.json.tar.gz']
+    country_code = 'ch'
diff --git a/kingfisher_scrapy/spiders/united_kingdom_digiwhist.py b/kingfisher_scrapy/spiders/united_kingdom_digiwhist.py
@@ -3,4 +3,4 @@
 
 class UnitedKingdomDigiwhist(DigiwhistBase):
     name = 'united_kingdom_digiwhist'
-    start_urls = ['https://opentender.eu/data/files/UK_ocds_data.json.tar.gz']
+    country_code = 'uk'