Merge pull request #1063 from open-contracting/1054-mexico-quien-es-q…

…uien-fixes fix(mexico_quien_es_quien*): remove records and fix releases
open-contracting · Apr 6, 2024 · 68acc63 · 68acc63
2 parents 61188e0 + 2654d3b
commit 68acc63
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 67 deletions.
diff --git a/docs/history.rst b/docs/history.rst
@@ -1,7 +1,12 @@
+Removed spiders
+===============
+
+This page records the spiders removed from Kingfisher Collect.
+
 Lapsed spiders
-==============
+--------------
 
-This page records the spiders that were available but stopped publishing and therefore where removed from Kingfisher Collect, since January 2022:
+Spiders for publications that were available but stopped publishing, since January 2022:
 
 - 2024-01-09: `mexico_puebla_itaipue, nigeria_edo_state <https://github.com/open-contracting/kingfisher-collect/pull/1047>`__
 - 2023-10-04: `honduras_cost, kenya_makueni, kyrgyzstan, portugal_bulk <https://github.com/open-contracting/kingfisher-collect/pull/1030>`__
@@ -12,3 +17,10 @@ This page records the spiders that were available but stopped publishing and the
 .. note::
 
    Since January 2022, any spider that stops working for more than six months will be deleted.
+
+Broken spiders
+--------------
+
+Spiders for publications that became broken, since April 2024:
+
+- 2024-04-05: `mexico_quien_es_quien_records <https://github.com/open-contracting/kingfisher-collect/pull/1063>`__ (pagination is broken)
diff --git a/docs/spiders.rst b/docs/spiders.rst
@@ -796,13 +796,6 @@ Mexico
 
    scrapy crawl mexico_nuevo_leon_releases
 
-.. autoclass:: kingfisher_scrapy.spiders.mexico_quien_es_quien_records.MexicoQuienEsQuienRecords
-   :no-members:
-
-.. code-block:: bash
-
-   scrapy crawl mexico_quien_es_quien_records
-
 .. autoclass:: kingfisher_scrapy.spiders.mexico_quien_es_quien_releases.MexicoQuienEsQuienReleases
    :no-members:
 

diff --git a/kingfisher_scrapy/spiders/mexico_quien_es_quien_base.py b/kingfisher_scrapy/spiders/mexico_quien_es_quien_base.py
diff --git a/kingfisher_scrapy/spiders/mexico_quien_es_quien_records.py b/kingfisher_scrapy/spiders/mexico_quien_es_quien_records.py
diff --git a/kingfisher_scrapy/spiders/mexico_quien_es_quien_releases.py b/kingfisher_scrapy/spiders/mexico_quien_es_quien_releases.py
@@ -1,22 +1,39 @@
-from kingfisher_scrapy.spiders.mexico_quien_es_quien_base import MexicoQuienEsQuienBase
+from kingfisher_scrapy.base_spiders import IndexSpider, PeriodicSpider
+from kingfisher_scrapy.util import parameters
 
 
-class MexicoQuienEsQuienReleases(MexicoQuienEsQuienBase):
+class MexicoQuienEsQuienReleases(IndexSpider, PeriodicSpider):
     """
     Domain
       QuiénEsQuién.Wiki
+    Spider arguments
+      from_date
+        Download only data from this date onward (YYYY-MM-DD format). Defaults to '1999-01-01'.
+      until_date
+        Download only data until this date (YYYY-MM-DD format). Defaults to '2021-12-31'.
     API documentation
       https://qqwapi-elastic.readthedocs.io/es/latest/
     Swagger API documentation
       https://api.quienesquien.wiki/v3/docs/
     """
     name = 'mexico_quien_es_quien_releases'
+    download_delay = 1
 
     # BaseSpider
+    default_from_date = '1999-01-01'
+    default_until_date = '2021-12-31'
+    date_format = 'date'
     root_path = 'data.item'
 
     # SimpleSpider
     data_type = 'release'
 
     # IndexSpider
-    base_url = 'https://api.quienesquien.wiki/v3/contracts?sort=date&sort_direction=desc'
+    result_count_pointer = '/count'
+    limit = 1000
+
+    # PeriodicSpider
+    formatter = staticmethod(parameters('start_date_min', 'start_date_max', 'offset'))
+    pattern = 'https://api.quienesquien.wiki/v3/contracts?start_date_min={0:%Y-%m-%d}&start_date_max={1:%Y-%m-%d}' \
+              f'&offset=0&limit={limit}'
+    start_requests_callback = 'parse_list'