diff --git a/kingfisher_scrapy/spiders/armenia.py b/kingfisher_scrapy/spiders/armenia.py index ca057b1e6..161eec3a0 100644 --- a/kingfisher_scrapy/spiders/armenia.py +++ b/kingfisher_scrapy/spiders/armenia.py @@ -5,6 +5,11 @@ class Armenia(LinksSpider): + """ + Spider arguments + sample + Download only the first release package in the dataset. + """ name = 'armenia' data_type = 'release_package' next_pointer = '/next_page/uri' diff --git a/kingfisher_scrapy/spiders/australia.py b/kingfisher_scrapy/spiders/australia.py index 374747230..593cd3a72 100644 --- a/kingfisher_scrapy/spiders/australia.py +++ b/kingfisher_scrapy/spiders/australia.py @@ -7,6 +7,15 @@ class Australia(LinksSpider): + """ + API documentation + https://data.gov.au/dataset/ds-dga-5c7fa69b-b0e9-4553-b8df-2a022dd2e982/distribution/dist-dga-a7f471ad-e085-49b5-bd6b-1b270ea46e99/details?q= + Swagger API documentation + https://app.swaggerhub.com/apis/austender/ocds-api/1.1#/ + Spider arguments + sample + Download only data released on 2018. + """ name = 'australia' data_type = 'release_package' next_page_formatter = staticmethod(parameters('cursor')) diff --git a/kingfisher_scrapy/spiders/australia_nsw.py b/kingfisher_scrapy/spiders/australia_nsw.py index 914e26cdc..a09ca825c 100644 --- a/kingfisher_scrapy/spiders/australia_nsw.py +++ b/kingfisher_scrapy/spiders/australia_nsw.py @@ -5,6 +5,13 @@ class AustraliaNSW(SimpleSpider): + """ + API documentation + https://github.com/NSW-eTendering/NSW-eTendering-API/blob/master/README.md + Spider arguments + sample + Download only 30 releases. + """ name = 'australia_nsw' data_type = 'release_package' diff --git a/kingfisher_scrapy/spiders/chile_compra_bulk.py b/kingfisher_scrapy/spiders/chile_compra_bulk.py index b35a9d137..54fb273e7 100644 --- a/kingfisher_scrapy/spiders/chile_compra_bulk.py +++ b/kingfisher_scrapy/spiders/chile_compra_bulk.py @@ -5,6 +5,13 @@ class ChileCompraBulk(ZipSpider): + """ + Bulk download documentation + https://desarrolladores.mercadopublico.cl/OCDS/DescargaMasiva + Spider arguments + sample + Download only data released on February 2017. + """ name = 'chile_compra_bulk' data_type = 'record_package' diff --git a/kingfisher_scrapy/spiders/chile_compra_records.py b/kingfisher_scrapy/spiders/chile_compra_records.py index 14c5d99cf..a40596233 100644 --- a/kingfisher_scrapy/spiders/chile_compra_records.py +++ b/kingfisher_scrapy/spiders/chile_compra_records.py @@ -3,6 +3,13 @@ class ChileCompraRecords(ChileCompraBaseSpider): + """ + Swagger API documentation + https://desarrolladores.mercadopublico.cl/docs/services/5a9ed02f718ed712f4403e75/operations/data-listaa-omes-agno-mes? + Spider arguments + sample + Download only data released on October 2017. + """ name = 'chile_compra_records' data_type = 'record_package' diff --git a/kingfisher_scrapy/spiders/chile_compra_releases.py b/kingfisher_scrapy/spiders/chile_compra_releases.py index a34673058..3ff8478f6 100644 --- a/kingfisher_scrapy/spiders/chile_compra_releases.py +++ b/kingfisher_scrapy/spiders/chile_compra_releases.py @@ -3,6 +3,13 @@ class ChileCompraReleases(ChileCompraBaseSpider): + """ + Swagger API documentation + https://desarrolladores.mercadopublico.cl/docs/services/5a9ed02f718ed712f4403e75/operations/data-listaa-omes-agno-mes? + Spider arguments + sample + Download only data released on October 2017. + """ name = 'chile_compra_releases' data_type = 'release_package' diff --git a/kingfisher_scrapy/spiders/colombia.py b/kingfisher_scrapy/spiders/colombia.py index 92a97df04..b6c7788ee 100644 --- a/kingfisher_scrapy/spiders/colombia.py +++ b/kingfisher_scrapy/spiders/colombia.py @@ -9,6 +9,19 @@ class Colombia(LinksSpider): + """ + API documentation + https://www.colombiacompra.gov.co/transparencia/api + Swagger API documentation + https://apiocds.colombiacompra.gov.co:8443/apiCCE2.0/ + Spider arguments + sample + Download only the first page of results. + page + The page number from which to start crawling. + year + The year to crawl. See API documentation for valid values. + """ name = 'colombia' next_page_formatter = staticmethod(parameters('page')) diff --git a/kingfisher_scrapy/spiders/honduras_portal_records.py b/kingfisher_scrapy/spiders/honduras_portal_records.py index b30444271..33ba9d976 100644 --- a/kingfisher_scrapy/spiders/honduras_portal_records.py +++ b/kingfisher_scrapy/spiders/honduras_portal_records.py @@ -5,6 +5,15 @@ class HondurasPortalRecords(LinksSpider): + """ + API documentation + http://www.contratacionesabiertas.gob.hn/manual_api/ + Swagger API documentation + http://www.contratacionesabiertas.gob.hn/servicio/ + Spider arguments + sample + Download only the first record package in the dataset. + """ name = 'honduras_portal_records' data_type = 'record_package' data_pointer = '/recordPackage' diff --git a/kingfisher_scrapy/spiders/honduras_portal_releases.py b/kingfisher_scrapy/spiders/honduras_portal_releases.py index 4cedf8ad1..71582b4aa 100644 --- a/kingfisher_scrapy/spiders/honduras_portal_releases.py +++ b/kingfisher_scrapy/spiders/honduras_portal_releases.py @@ -5,6 +5,15 @@ class HondurasPortalReleases(LinksSpider): + """ + API documentation + http://www.contratacionesabiertas.gob.hn/manual_api/ + Swagger API documentation + http://www.contratacionesabiertas.gob.hn/servicio/ + Spider arguments + sample + Download only the first release package in the dataset. + """ name = 'honduras_portal_releases' data_type = 'release_package' data_pointer = '/releasePackage' diff --git a/kingfisher_scrapy/spiders/kenya_makueni.py b/kingfisher_scrapy/spiders/kenya_makueni.py index 771572137..4d44f9514 100644 --- a/kingfisher_scrapy/spiders/kenya_makueni.py +++ b/kingfisher_scrapy/spiders/kenya_makueni.py @@ -7,6 +7,13 @@ class KenyaMakueni(SimpleSpider): + """ + Swagger API documentation + https://opencontracting.makueni.go.ke/swagger-ui.html#/ocds-controller + Spider arguments + sample + Download only the first 10 release packages in the dataset. + """ name = 'kenya_makueni' data_type = 'release_package_list' step = 10 diff --git a/kingfisher_scrapy/spiders/malta.py b/kingfisher_scrapy/spiders/malta.py index 877caf7fe..85c0cf10e 100644 --- a/kingfisher_scrapy/spiders/malta.py +++ b/kingfisher_scrapy/spiders/malta.py @@ -8,6 +8,13 @@ class Malta(ZipSpider): + """ + API documentation + https://docs.google.com/document/d/1VnCEywKkkQ7BcVbT7HlW2s_N_QI8W0KE/edit + Spider arguments + sample + Download only data released on October 2019. + """ name = 'malta' data_type = 'record_package' diff --git a/kingfisher_scrapy/spiders/mexico_quien_es_quien.py b/kingfisher_scrapy/spiders/mexico_quien_es_quien.py index c2c7dad6e..f4ba4435c 100644 --- a/kingfisher_scrapy/spiders/mexico_quien_es_quien.py +++ b/kingfisher_scrapy/spiders/mexico_quien_es_quien.py @@ -8,6 +8,15 @@ class MexicoQuienEsQuien(BaseSpider): + """ + API documentation + https://quienesquienapi.readthedocs.io/es/latest/ + Swagger API documentation + https://api.quienesquien.wiki/v2/docs/ + Spider arguments + sample + Download a single record package with 10 records. + """ name = 'mexico_quien_es_quien' download_delay = 0.9 diff --git a/kingfisher_scrapy/spiders/nepal_dhangadhi.py b/kingfisher_scrapy/spiders/nepal_dhangadhi.py index 66c9e88a1..70830b016 100644 --- a/kingfisher_scrapy/spiders/nepal_dhangadhi.py +++ b/kingfisher_scrapy/spiders/nepal_dhangadhi.py @@ -7,6 +7,13 @@ class NepalDhangadhi(SimpleSpider): + """ + Bulk download documentation + https://ims.susasan.org/dhangadhi/about + Spider arguments + sample + Download only the first release package in the dataset. + """ name = 'nepal_dhangadhi' data_type = 'release_package' diff --git a/kingfisher_scrapy/spiders/nepal_portal.py b/kingfisher_scrapy/spiders/nepal_portal.py index 3b91a8761..ee138575d 100644 --- a/kingfisher_scrapy/spiders/nepal_portal.py +++ b/kingfisher_scrapy/spiders/nepal_portal.py @@ -5,6 +5,13 @@ class NepalPortal(SimpleSpider): + """ + Bulk download documentation + http://ppip.gov.np/downloads + Spider arguments + sample + Download only data released on 2018. + """ name = 'nepal_portal' data_type = 'release_package' diff --git a/kingfisher_scrapy/spiders/nigeria_portal.py b/kingfisher_scrapy/spiders/nigeria_portal.py index d108cd650..ebb26d4d0 100644 --- a/kingfisher_scrapy/spiders/nigeria_portal.py +++ b/kingfisher_scrapy/spiders/nigeria_portal.py @@ -5,6 +5,11 @@ class NigeriaPortal(SimpleSpider): + """ + Spider arguments + sample + Download only the first release package in the dataset. + """ name = 'nigeria_portal' data_type = 'release_package' diff --git a/kingfisher_scrapy/spiders/paraguay_hacienda.py b/kingfisher_scrapy/spiders/paraguay_hacienda.py index 244b63a8f..2eb8a1924 100644 --- a/kingfisher_scrapy/spiders/paraguay_hacienda.py +++ b/kingfisher_scrapy/spiders/paraguay_hacienda.py @@ -9,6 +9,18 @@ class ParaguayHacienda(BaseSpider): + """ + Swagger API documentation + https://datos.hacienda.gov.py/odmh-api-v1/api-docs/ + Spider arguments + sample + Download only 11 releases. + Environment variables + KINGFISHER_PARAGUAY_HACIENDA_REQUEST_TOKEN + To get an API account and request token go to https://datos.hacienda.gov.py/aplicaciones/new. + KINGFISHER_PARAGUAY_HACIENDA_CLIENT_SECRET + Your client secret generated. + """ name = 'paraguay_hacienda' start_time = None diff --git a/kingfisher_scrapy/spiders/portugal.py b/kingfisher_scrapy/spiders/portugal.py index b5839e1ec..cc19db424 100644 --- a/kingfisher_scrapy/spiders/portugal.py +++ b/kingfisher_scrapy/spiders/portugal.py @@ -7,6 +7,15 @@ class Portugal(ZipSpider): + """ + API documentation + https://dados.gov.pt/pt/apidoc/ + Swagger API documentation + https://dados.gov.pt/pt/apidoc/#/datasets + Spider arguments + sample + Download only the first file in the dataset. + """ name = 'portugal' data_type = 'record_package' encoding = 'iso-8859-1' diff --git a/kingfisher_scrapy/spiders/uganda_releases.py b/kingfisher_scrapy/spiders/uganda_releases.py index ad3c35824..57f547d15 100644 --- a/kingfisher_scrapy/spiders/uganda_releases.py +++ b/kingfisher_scrapy/spiders/uganda_releases.py @@ -7,6 +7,13 @@ class Uganda(SimpleSpider): + """ + API documentation + https://docs.google.com/spreadsheets/d/10tVioy-VOQa1FwWoRl5e1pMbGpiymA0iycNcoDFkvks/edit#gid=365266172 + Spider arguments + sample + Download only 19 releases. + """ name = 'uganda_releases' data_type = 'release_package' diff --git a/kingfisher_scrapy/spiders/uruguay_historical.py b/kingfisher_scrapy/spiders/uruguay_historical.py index 057c23588..b01b24f9d 100644 --- a/kingfisher_scrapy/spiders/uruguay_historical.py +++ b/kingfisher_scrapy/spiders/uruguay_historical.py @@ -3,6 +3,13 @@ class UruguayHistorical(ZipSpider): + """ + Bulk download documentation + https://www.gub.uy/agencia-compras-contrataciones-estado/datos-y-estadisticas/datos/open-contracting + Spider arguments + sample + Download only data released on 2002. + """ name = 'uruguay_historical' data_type = 'release_package' diff --git a/kingfisher_scrapy/spiders/uruguay_records.py b/kingfisher_scrapy/spiders/uruguay_records.py index 2c629ff7e..b3053ed4e 100644 --- a/kingfisher_scrapy/spiders/uruguay_records.py +++ b/kingfisher_scrapy/spiders/uruguay_records.py @@ -3,6 +3,13 @@ class UruguayRecords(UruguayBase): + """ + API documentation + https://www.gub.uy/agencia-compras-contrataciones-estado/datos-y-estadisticas/datos/open-contracting + Spider arguments + sample + Download only 1 record. + """ name = 'uruguay_records' data_type = 'record_package' diff --git a/kingfisher_scrapy/spiders/uruguay_releases.py b/kingfisher_scrapy/spiders/uruguay_releases.py index 70dee3e79..2b867e4a8 100644 --- a/kingfisher_scrapy/spiders/uruguay_releases.py +++ b/kingfisher_scrapy/spiders/uruguay_releases.py @@ -3,6 +3,13 @@ class UruguayReleases(UruguayBase): + """ + API documentation + https://www.gub.uy/agencia-compras-contrataciones-estado/datos-y-estadisticas/datos/open-contracting + Spider arguments + sample + Download only 1 release. + """ name = 'uruguay_releases' data_type = 'release_package' diff --git a/kingfisher_scrapy/spiders/zambia.py b/kingfisher_scrapy/spiders/zambia.py index be15337b3..614827200 100644 --- a/kingfisher_scrapy/spiders/zambia.py +++ b/kingfisher_scrapy/spiders/zambia.py @@ -7,6 +7,11 @@ class Zambia(ZipSpider): + """ + Spider arguments + sample + Download only data released on July 2016. + """ name = 'zambia' data_type = 'record_package'