Merge branch 'main' of github.com:open-contracting/kingfisher-collect

kingfisher_scrapy/base_spiders/base_spider.py

@@ -283,6 +283,7 @@ def build_request(self, url, formatter, **kwargs):
             assert kwargs['meta']['file_name']
         else:
             meta['file_name'] = formatter(url)
+            # Other extensions are related to the Unflatten pipeline and CompressedFileSpider base class.
             if not meta['file_name'].endswith(('.json', '.csv', '.xlsx', '.rar', '.zip')):
                 meta['file_name'] += '.json'
         if 'meta' in kwargs:

kingfisher_scrapy/spiders/ecuador_sercop_bulk.py

@@ -1,5 +1,5 @@
 from kingfisher_scrapy.base_spiders import CompressedFileSpider, PeriodicSpider
-from kingfisher_scrapy.util import components
+from kingfisher_scrapy.util import components, join
 
 
 class EcuadorSERCOPBulk(CompressedFileSpider, PeriodicSpider):
@@ -29,7 +29,4 @@ class EcuadorSERCOPBulk(CompressedFileSpider, PeriodicSpider):
     # PeriodicSpider
     pattern = 'https://datosabiertos.compraspublicas.gob.ec/PLATAFORMA/download'\
               '?type=json&year={0:%Y}&month={0:%m}&method=all'
-    formatter = staticmethod(components(-1))
-
-    def build_request(self, url, formatter, **kwargs):
-        return super().build_request(url, formatter, meta={'file_name': f'{formatter(url)}.zip'}, **kwargs)
+    formatter = staticmethod(join(components(-1), extension='zip'))

kingfisher_scrapy/util.py

@@ -56,7 +56,7 @@ def wrapper(url):
 
 def join(*functions, extension=None):
     """
-    Returns a function that joins the given functions' outputs.
+    Returns a function that joins the given functions' outputs and sets the file extension, if provided.
 
     >>> join(components(-1), parameters('page'))('http://example.com/api/planning.json?page=1')
     'planning-page-1'