malta: Order requests to get most recent data first #451

open-contracting · Mar 26, 2021 · a3c4cbe · a3c4cbe
1 parent f06fd7e
commit a3c4cbe
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 1 deletion.
diff --git a/kingfisher_scrapy/spiders/france.py b/kingfisher_scrapy/spiders/france.py
@@ -16,6 +16,7 @@ class France(BigFileSpider):
     def start_requests(self):
         # A CKAN API JSON response.
         # Ministère de l'économie, des finances et de la relance
+        # https://www.data.gouv.fr/fr/datasets/donnees-essentielles-de-la-commande-publique-fichiers-consolides/
         url = 'https://www.data.gouv.fr/api/1/datasets/donnees-essentielles-de-la-commande-publique-fichiers' \
               '-consolides/'
         yield scrapy.Request(url, meta={'file_name': 'page-1.json'}, callback=self.parse_list)

diff --git a/kingfisher_scrapy/spiders/malta.py b/kingfisher_scrapy/spiders/malta.py
@@ -30,7 +30,7 @@ def parse_list(self, response):
         urls = response.json()['packagesPerMonth']
 
         netloc = urlsplit(response.request.url).netloc
-        for url in urls:
+        for url in reversed(urls):
             # URL looks like http://malta-demo-server.eurodyn.com/ocds/services/recordpackage/getrecordpackage/2020/1
             yield self.build_request(urlsplit(url)._replace(netloc=netloc).geturl(),
                                      formatter=join(components(-2), extension='zip'))
diff --git a/kingfisher_scrapy/spiders/mexico_administracion_publica_federal_api.py b/kingfisher_scrapy/spiders/mexico_administracion_publica_federal_api.py
@@ -28,4 +28,5 @@ class MexicoAdministracionPublicaFederalAPI(IndexSpider):
 
     def start_requests(self):
         url = 'https://api.datos.gob.mx/v2/contratacionesabiertas'
+        # The pages are in reverse chronological order.
         yield scrapy.Request(url, meta={'file_name': 'page-1.json'}, callback=self.parse_list)