Merge branch 'main' into 934-ecuador-api

.github/workflows/nonlinux.yml

@@ -0,0 +1,22 @@
+name: CI (Non-Linux)
+on: [push, pull_request]
+jobs:
+  build:
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: 3.6
+          cache: pip
+          cache-dependency-path: '**/requirements*.txt'
+      # Don't install editable projects in the current working directory.
+      # https://pip.pypa.io/en/latest/reference/pip_install/#install-src
+      - run: pip install --src $GITHUB_WORKSPACE/../src -r requirements_dev.txt
+      - env:
+          CI_SKIP: true
+        run: pytest -W error -rs --cov kingfisher_scrapy

docs/contributing/index.rst

@@ -124,6 +124,7 @@ Since there are many class attributes that control a spider's behavior, please p
       next_pointer = '/next_page/uri'
 
       # PeriodicSpider
+      formatter = staticmethod(parameters('page'))
       pattern = 'https://example.com/{}'
       start_requests_callback = 'parse_list'
 
@@ -135,7 +136,7 @@ Since there are many class attributes that control a spider's behavior, please p
       start_page = 0
       formatter = staticmethod(parameters('pageNumber'))
       chronological_order = 'asc'
-      parse_list_callback = 'parse_custom'
+      parse_list_callback = 'parse_page'
       param_page = 'pageNumber'
       param_limit = 'customLimit'
       param_offset = = 'customOffset'

docs/local.rst

@@ -18,6 +18,14 @@ When ready, open a shell, and run:
    cd kingfisher-collect
    pip install -r requirements.txt
 
+.. tip::
+
+   If you encounter an error relating to ``psycopg2``, try instead:
+
+   .. code-block:: bash
+
+      pip install psycopg2-binary -r requirements.txt
+
 To extract data from RAR files, you must have the ``unrar`` or `unar <https://theunarchiver.com/command-line>`__ command-line utility.
 
 The next steps assume that you have changed to the ``kingfisher-collect`` directory (the ``cd`` command above).

kingfisher_scrapy/spiders/uganda_releases.py

@@ -39,14 +39,14 @@ class UgandaReleases(IndexSpider):
 
     # IndexSpider
     total_pages_pointer = '/data/last_page'
-    parse_list_callback = 'parse_data'
+    parse_list_callback = 'parse_page'
 
     def start_requests(self):
         url = 'https://gpppapi.com/adminapi/public/api/pdes'
         yield scrapy.Request(url, meta={'file_name': 'page-1.json'}, callback=self.parse_list)
 
     @handle_http_error
-    def parse_data(self, response):
+    def parse_page(self, response):
         pattern = 'https://gpppapi.com/adminapi/public/api/open-data/v1/releases/{tag}?fy={fy}&pde={pde}'
 
         for pdes in response.json()['data']['data']:

kingfisher_scrapy/spiders/united_kingdom_contracts_finder_base.py

@@ -14,7 +14,7 @@ class UnitedKingdomContractsFinderBase(IndexSpider):
     ocds_version = '1.0'  # uses deprecated fields
 
     # IndexSpider
-    parse_list_callback = 'build_urls'
+    parse_list_callback = 'parse_page'
     total_pages_pointer = '/maxPage'
 
     # Local

requirements.txt

@@ -123,7 +123,7 @@ rfc3986-validator==0.1.1
     # via -r requirements.in
 schema==0.7.2
     # via flattentool
-scrapyd==1.2.1
+scrapyd==1.3.0
     # via -r requirements.in
 scrapyd-client==1.2.0
     # via -r requirements.in

requirements.txt.sha256

@@ -1 +1 @@
-7194e608596c11ad70f54e5c7d5cd408c5708b1852be8dad0045a71cb8d52d85  requirements.txt
+ee2d96e2677c4f4540df1759d3a455d4de961fdf0f92d1c39285891a91faea5c  requirements.txt

requirements_dev.in

@@ -5,6 +5,7 @@ flake8
 isort
 openpyxl
 pip-tools
+psycopg2-binary
 pytest
 pytest-cov
 pytest-order

requirements_dev.txt

@@ -168,6 +168,8 @@ pluggy==0.13.1
     # via pytest
 psycopg2==2.9.2
     # via -r requirements.txt
+psycopg2-binary==2.9.2
+    # via -r requirements_dev.in
 py==1.10.0
     # via pytest
 pyasn1==0.4.8
@@ -234,7 +236,7 @@ schema==0.7.2
     # via
     #   -r requirements.txt
     #   flattentool
-scrapyd==1.2.1
+scrapyd==1.3.0
     # via -r requirements.txt
 scrapyd-client==1.2.0
     # via -r requirements.txt

tests/extensions/test_database_store.py

@@ -11,7 +11,7 @@
 from tests import spider_with_crawler
 
 database_url = os.getenv('KINGFISHER_COLLECT_DATABASE_URL')
-skip_test_if = not database_url and not os.getenv('CI')
+skip_test_if = not database_url and ('CI' not in os.environ or 'CI_SKIP' in os.environ)
 
 
 def test_from_crawler_missing_arguments():

tests/extensions/test_kingfisher_process_api2.py

@@ -12,7 +12,7 @@
 from tests import ExpectedError, spider_with_crawler, spider_with_files_store
 
 rabbit_url = os.getenv('RABBIT_URL')
-skip_test_if = not rabbit_url and not os.getenv('CI')
+skip_test_if = not rabbit_url and ('CI' not in os.environ or 'CI_SKIP' in os.environ)
 
 items_scraped = [
     ('build_file', 'file.json', {