Merge branch 'main' into 696_uruguay_historical

kingfisher_scrapy/commands/checkall.py

@@ -118,10 +118,15 @@ def check(self):
             if not hasattr(self.cls, class_attribute) and spider_argument in spider_arguments:
                 self.log('warning', f'unexpected "{spider_argument}" spider argument ({class_attribute} is not set)')
 
-        self.check_date_spider_argument('from_date', spider_arguments, lambda cls: repr(cls.default_from_date),
+        def default_from_date(cls):
+            return repr(getattr(cls, 'default_from_date', None))
+
+        self.check_date_spider_argument('from_date', spider_arguments, default_from_date,
                                         'Download only data from this {period} onward ({format} format).')
 
-        def default(cls):
+        def default_until_date(cls):
+            if hasattr(cls, 'default_until_date'):
+                return f"'{cls.default_until_date}'"
             if cls.date_format == 'datetime':
                 return 'now'
             elif cls.date_format == 'date':
@@ -131,7 +136,7 @@ def default(cls):
             elif cls.date_format == 'year':
                 return 'the current year'
 
-        self.check_date_spider_argument('until_date', spider_arguments, default,
+        self.check_date_spider_argument('until_date', spider_arguments, default_until_date,
                                         'Download only data until this {period} ({format} format).')
 
     def check_list(self, items, known_items, name):
@@ -152,16 +157,17 @@ def check_list(self, items, known_items, name):
     def check_date_spider_argument(self, spider_argument, spider_arguments, default, format_string):
         if spider_argument in spider_arguments:
             # These classes are known to have more specific semantics.
-            if self.cls.__name__ in ('PortugalRecords', 'PortugalReleases', 'ScotlandPublicContracts'):
+            if self.cls.__name__ in ('ColombiaBulk', 'PortugalRecords', 'PortugalReleases',
+                                     'ScotlandPublicContracts'):
                 level = 'info'
             else:
                 level = 'warning'
 
             if self.cls.date_required:
                 format_string += " Defaults to {default}."
-            elif spider_argument == 'from_date':
+            elif spider_argument == 'from_date' and 'until_date' in spider_arguments:
                 format_string += "\n  If ``until_date`` is provided, defaults to {default}."
-            elif spider_argument == 'until_date':
+            elif spider_argument == 'until_date' and 'from_date' in spider_arguments:
                 format_string += "\n  If ``from_date`` is provided, defaults to {default}."
 
             if self.cls.date_format == 'datetime':

kingfisher_scrapy/spiders/afghanistan_record_packages.py

@@ -7,9 +7,11 @@ class AfghanistanRecordPackages(AfghanistanPackagesBase):
       Afghanistan Government Electronic & Open Procurement System (AGEOPS)
     Spider arguments
       from_date
-        Download only data from this date onward (YYYY-MM-DD format). Defaults to '2018-12-15'.
+        Download only data from this date onward (YYYY-MM-DD format).
+        If ``until_date`` is provided, defaults to '2018-12-15'.
       until_date
-        Download only data until this date (YYYY-MM-DD format). Defaults to today.
+        Download only data until this date (YYYY-MM-DD format).
+        If ``from_date`` is provided, defaults to today.
     API documentation
       https://ocds.ageops.net/
     """

kingfisher_scrapy/spiders/afghanistan_release_packages.py

@@ -7,9 +7,11 @@ class AfghanistanReleasePackages(AfghanistanPackagesBase):
       Afghanistan Government Electronic & Open Procurement System (AGEOPS)
     Spider arguments
       from_date
-        Download only data from this date onward (YYYY-MM-DD format). Defaults to '2018-12-15'.
+        Download only data from this date onward (YYYY-MM-DD format).
+        If ``until_date`` is provided, defaults to '2018-12-15'.
       until_date
-        Download only data until this date (YYYY-MM-DD format). Defaults to today.
+        Download only data until this date (YYYY-MM-DD format).
+        If ``from_date`` is provided, defaults to today.
     API documentation
       https://ocds.ageops.net/
     """

kingfisher_scrapy/spiders/afghanistan_releases.py

@@ -12,9 +12,11 @@ class AfghanistanReleases(SimpleSpider):
       Afghanistan Government Electronic & Open Procurement System (AGEOPS)
     Spider arguments
       from_date
-        Download only data from this date onward (YYYY-MM-DD format). Defaults to '2018-12-12'.
+        Download only data from this date onward (YYYY-MM-DD format).
+        If ``until_date`` is provided, defaults to '2018-12-12'.
       until_date
-        Download only data until this date (YYYY-MM-DD format). Defaults to today.
+        Download only data until this date (YYYY-MM-DD format).
+        If ``from_date`` is provided, defaults to today.
     API documentation
       https://ocds.ageops.net/
     """

kingfisher_scrapy/spiders/australia.py

@@ -10,9 +10,9 @@ class Australia(LinksSpider):
       AusTender
     Spider arguments
       from_date
-        Download only contracts from this time onward (YYYY-MM-DDThh:mm:ss format). Defaults to '2004-01-01T00:00:00'.
+        Download only data from this time onward (YYYY-MM-DDThh:mm:ss format). Defaults to '2004-01-01T00:00:00'.
       until_date
-        Download only contracts until this time (YYYY-MM-DDThh:mm:ss format). Defaults to now.
+        Download only data until this time (YYYY-MM-DDThh:mm:ss format). Defaults to now.
     API documentation
       https://github.com/austender/austender-ocds-api
     Swagger API documentation

kingfisher_scrapy/spiders/costa_rica_poder_judicial_records.py

@@ -10,9 +10,11 @@ class CostaRicaPoderJudicialRecords(SimpleSpider):
       Poder Judicial de Costa Rica
     Spider arguments
       from_date
-        Download only data from this year onward (YYYY format). Defaults to '2018'.
+        Download only data from this year onward (YYYY format).
+        If ``until_date`` is provided, defaults to '2018'.
       until_date
-        Download only data until this year (YYYY format). Defaults to the current year.
+        Download only data until this year (YYYY format).
+        If ``from_date`` is provided, defaults to the current year.
     Bulk download documentation
       http://datosabiertospj.eastus.cloudapp.azure.com/dataset/estandar-de-datos-de-contrataciones-abiertas-ocds
     """

kingfisher_scrapy/spiders/indonesia_opentender.py

@@ -8,11 +8,9 @@ class IndonesiaOpentender(CompressedFileSpider, PeriodicSpider):
          Open Tender
        Spider arguments
          from_date
-           Download only releases from this date onward (YYYY format).
-           If ``from_date`` is not provided defaults to 2008.
+           Download only data from this year onward (YYYY format). Defaults to '2008'.
          until_date
-           Download only releases until this date (YYYY format).
-           If ``until_date`` is not provided defaults to current year.
+           Download only data until this year (YYYY format). Defaults to the current year.
        Bulk download documentation
          https://v3.opentender.net/#/ocds
     """

kingfisher_scrapy/spiders/malta.py

@@ -13,9 +13,11 @@ class Malta(CompressedFileSpider):
       Malta
     Spider arguments
       from_date
-        Download only data from this month onward (YYYY-MM format). Defaults to '2019-10'.
+        Download only data from this month onward (YYYY-MM format).
+        If ``until_date`` is provided, defaults to '2019-10'.
       until_date
-        Download only data until this month (YYYY-MM format). Defaults to the current month.
+        Download only data until this month (YYYY-MM format).
+        If ``from_date`` is provided, defaults to the current month.
     API documentation
       https://docs.google.com/document/d/1VnCEywKkkQ7BcVbT7HlW2s_N_QI8W0KE/edit
     """

kingfisher_scrapy/spiders/mexico_inai_api.py

@@ -8,11 +8,9 @@ class MexicoINAIAPI(PeriodicSpider):
       Instituto Nacional de Transparencia, Acceso a la Información y Protección de Datos Personales (INAI)
     Spider arguments
       from_date
-        Download only releases from this date onward (YYYY format).
-        If ``from_date`` is not provided defaults to 2015.
+        Download only data from this year onward (YYYY format). Defaults to '2015'.
       until_date
-        Download only releases until this date (YYYY format).
-        If ``until_date`` is not provided defaults to 2020.
+        Download only data until this year (YYYY format). Defaults to '2020'.
     API documentation
       http://contratacionesabiertas.inai.org.mx/contratacionesabiertas/datosabiertos
     """

kingfisher_scrapy/spiders/mexico_nuevo_leon_records.py

@@ -1,7 +1,6 @@
 import scrapy
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import components, handle_http_error
 
 
 class MexicoNuevoLeonRecords(SimpleSpider):
@@ -18,14 +17,6 @@ class MexicoNuevoLeonRecords(SimpleSpider):
 
     def start_requests(self):
         yield scrapy.Request(
-            'http://si.nl.gob.mx/siasi_ws/api/ocds/DescargarRecordPackage?usuario=undefined&servidor=undefined',
-            meta={'file_name': 'list.json'},
-            callback=self.parse_list
+            'http://si.nl.gob.mx/siasi_ws/api/ocds/DescargarRecordPackage',
+            meta={'file_name': 'records.json'}
         )
-
-    @handle_http_error
-    def parse_list(self, response):
-        content = response.json()
-        url = content['urlWebOcdspackage']
-        if url:
-            yield self.build_request(url, formatter=components(-1))

kingfisher_scrapy/spiders/mexico_nuevo_leon_releases.py

@@ -1,7 +1,8 @@
-from kingfisher_scrapy.spiders.mexico_nuevo_leon_base import MexicoNuevoLeonBase
+from kingfisher_scrapy.base_spider import CompressedFileSpider
+from kingfisher_scrapy.util import components
 
 
-class MexicoNuevoLeonReleases(MexicoNuevoLeonBase):
+class MexicoNuevoLeonReleases(CompressedFileSpider):
     """
     Domain
       Secretaría de Infraestructura del Gobierno del Estado de Nuevo León
@@ -18,3 +19,9 @@ class MexicoNuevoLeonReleases(MexicoNuevoLeonBase):
 
     # CompressedFileSpider
     file_name_must_contain = 'ReleasePackage'
+
+    def start_requests(self):
+        yield self.build_request(
+            'http://si.nl.gob.mx/acceso/DatosAbiertos/JSONsInfraestructuraAbierta.rar',
+            formatter=components(-1)
+        )

kingfisher_scrapy/spiders/mexico_yucatan_inaip.py

@@ -9,11 +9,9 @@ class MexicoYucatan(PeriodicSpider):
       (INAIP) - Yucatán
     Spider arguments
       from_date
-        Download only releases from this date onward (YYYY format).
-        If ``from_date`` is not provided defaults to 2020.
+        Download only data from this year onward (YYYY format). Defaults to '2020'.
       until_date
-        Download only releases until this date (YYYY format).
-        If ``until_date`` is not provided defaults to 2020.
+        Download only data until this year (YYYY format). Defaults to '2020'.
     API documentation
       https://contratacionesabiertas.inaipyucatan.org.mx/contratacionesabiertas/datosabiertos#
     """

kingfisher_scrapy/spiders/moldova.py

@@ -8,9 +8,15 @@ class Moldova(SimpleSpider):
     """
     Domain
       MTender
+    Spider arguments
+      from_date
+        Download only data from this time onward (YYYY-MM-DDThh:mm:ss format).
     """
     name = 'moldova'
 
+    # BaseSpider
+    date_format = 'datetime'
+
     # SimpleSpider
     data_type = 'release_package'
 
@@ -25,7 +31,9 @@ def start_requests(self):
         #
         # Note: The OCIDs from the /budgets/ endpoint have no corresponding data in the second service. The OCIDs from
         # the /tenders/plan/ endpoint are the same as from the /tenders/ endpoint.
-        url = 'https://public.mtender.gov.md/tenders/'
+        url = f'https://public.mtender.gov.md/tenders/'
+        if self.from_date:
+            url = f'{url}?offset={self.from_date.strftime(self.date_format)}'
         yield scrapy.Request(url, meta={'file_name': 'list.json'}, callback=self.parse_list)
 
     @handle_http_error

kingfisher_scrapy/spiders/moldova_old.py

@@ -8,9 +8,9 @@ class MoldovaOld(PeriodicSpider):
       Public Procurement Agency (PPA)
     Spider arguments
       from_date
-        Download only data from this month onward (YYYY format). Defaults to '2012'.
+        Download only data from this year onward (YYYY format). Defaults to '2012'.
       until_date
-        Download only data until this month (YYYY format). Defaults to '2018'.
+        Download only data until this year (YYYY format). Defaults to '2017'.
     Bulk download documentation
       http://opencontracting.date.gov.md/downloads
     """

kingfisher_scrapy/spiders/uruguay_historical.py

@@ -12,9 +12,9 @@ class UruguayHistorical(CompressedFileSpider):
       Agencia Reguladora de Compras Estatales (ARCE)
     Spider arguments
       from_date
-        Download only data from this month onward (YYYY format). Defaults to '2002'.
+        Download only data from this year onward (YYYY format). Defaults to '2002'.
       until_date
-        Download only data until this month (YYYY format).
+        Download only data until this year (YYYY format).
     Bulk download documentation
       https://www.gub.uy/agencia-compras-contrataciones-estado/datos-y-estadisticas/datos/open-contracting
     """

kingfisher_scrapy/spiders/uruguay_records.py

@@ -10,7 +10,7 @@ class UruguayRecords(UruguayBase):
       from_date
         Download only data from this month onward (YYYY-MM format). Defaults to '2017-11'.
       until_date
-        Download only data until this month (YYYY-MM format). Defaults to the current year-month.
+        Download only data until this month (YYYY-MM format). Defaults to the current month.
     API documentation
       https://www.gub.uy/agencia-compras-contrataciones-estado/datos-y-estadisticas/datos/open-contracting
     """

kingfisher_scrapy/spiders/uruguay_releases.py

@@ -10,7 +10,7 @@ class UruguayReleases(UruguayBase):
       from_date
         Download only data from this month onward (YYYY-MM format). Defaults to '2017-11'.
       until_date
-        Download only data until this month (YYYY-MM format). Defaults to the current year-month.
+        Download only data until this month (YYYY-MM format). Defaults to the current month.
     API documentation
       https://www.gub.uy/agencia-compras-contrataciones-estado/datos-y-estadisticas/datos/open-contracting
     """

kingfisher_scrapy/spiders/zambia.py

@@ -12,9 +12,11 @@ class Zambia(CompressedFileSpider):
       Zambia Public Procurement Authority
     Spider arguments
       from_date
-        Download only data from this month onward (YYYY-MM format). Defaults to '2016-07'.
+        Download only data from this month onward (YYYY-MM format).
+        If ``until_date`` is provided, defaults to '2016-07'.
       until_date
-        Download only data until this month (YYYY-MM format). Defaults to the current month.
+        Download only data until this month (YYYY-MM format).
+        If ``from_date`` is provided, defaults to the current month.
     """
     name = 'zambia'
     date_format = 'year-month'

requirements_dev.txt

@@ -165,7 +165,7 @@ protego==0.1.16
     # via
     #   -r requirements.txt
     #   scrapy
-py==1.8.0
+py==1.10.0
     # via pytest
 pyasn1-modules==0.2.7
     # via