Migrate CI to GH actions

.github/workflows/main.yml

@@ -1,31 +1,158 @@
-name: Run test suite
+name: Build
 on: [push, pull_request]
 
 jobs:
-  test-windows:
-    name: "Windows Tests"
+  build:
+    name: Build
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest]
-        python-version: [3.7, 3.8]
-        env: [TOXENV: py]
         include:
+        # non-testing jobs
+        - os: ubuntu-18.04
+          python-version: 3.8
+          env:
+            TOXENV: security
+        - os: ubuntu-18.04
+          python-version: 3.8
+          env:
+            TOXENV: flake8
+        - os: ubuntu-18.04
+          python-version: 3.8
+          env:
+            TOXENV: pylint
+        - os: ubuntu-18.04
+          python-version: 3.8
+          env:
+            TOXENV: typing
+        - os: ubuntu-18.04
+          python-version: 3.7  # Keep in sync with .readthedocs.yml
+          env:
+            TOXENV: docs
+
+        # tests
+        - os: ubuntu-18.04
+          python-version: 3.7
+          env:
+            TOXENV: py
+        - os: ubuntu-18.04
+          python-version: 3.8
+          env:
+            TOXENV: py
+        - os: ubuntu-18.04
+          python-version: pypy3
+          env:
+            TOXENV: pypy3
+            PYPY_VERSION: 3.6-v7.3.1
+
+        # windows tests
+        - os: windows-latest
+          python-version: 3.7
+          env:
+            TOXENV: py
+        - os: windows-latest
+          python-version: 3.8
+          env:
+            TOXENV: py
+
+        # pinned deps
+        - os: ubuntu-18.04
+          python-version: 3.6.12
+          env:
+            TOXENV: pinned
+        - os: ubuntu-18.04
+          python-version: 3.6.12
+          env:
+            TOXENV: asyncio-pinned
         - os: windows-latest
           python-version: 3.6
           env:
             TOXENV: windows-pinned
+        - os: ubuntu-18.04
+          python-version: pypy3
+          env:
+            TOXENV: pypy3-pinned
+            PYPY_VERSION: 3.6-v7.2.0
+
+        # extras
+        - os: ubuntu-18.04
+          python-version: 3.8
+          env:
+            TOXENV: extra-deps
+        - os: ubuntu-18.04
+          python-version: 3.8
+          env:
+            TOXENV: asyncio
 
     steps:
     - uses: actions/checkout@v2
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Run test suite
+    - name: Install system libraries
+      if: startsWith(matrix.os, 'ubuntu') && (matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned'))
+      run: |
+        sudo apt-get update
+        sudo apt-get install libxml2-dev libxslt-dev
+
+    - name: Run check (windows)
+      if: startsWith(matrix.os, 'windows')
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox
+        tox
+
+    - name: Run check (ubuntu)
+      id: ubuntu-tests
+      if: startsWith(matrix.os, 'ubuntu')
       env: ${{ matrix.env }}
       run: |
-        pip install -U tox twine wheel codecov
+        if [[ ! -z "$PYPY_VERSION" ]]; then
+          export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+          wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
+          tar -jxf ${PYPY_VERSION}.tar.bz2
+          $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
+          source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+        fi
+        pip install -U tox
         tox
+        if [ -f "coverage.xml" ]; then
+          echo ::set-output name=upload_coverage_report::true
+        fi
+
+    - name: Upload coverage report
+      if: steps.ubuntu-tests.outputs.upload_coverage_report == 'true'
+      run: bash <(curl -s https://codecov.io/bash)
+
+  publish:
+    name: "Publish package to PyPI"
+    runs-on: ubuntu-18.04
+    needs: [build]
+    if: startsWith(github.event.ref, 'refs/tags/')
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Check Tag
+      id: check-release-tag
+      run: |
+        if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
+          echo ::set-output name=release_tag::true
+        fi
+
+    - name: Publish to PyPI
+      if: steps.check-release-tag.outputs.release_tag == 'true'
+      run: |
+        pip install --upgrade setuptools wheel twine
+        python setup.py sdist bdist_wheel
+        export TWINE_USERNAME=__token__
+        export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
+        twine upload dist/*

setup.py

@@ -24,7 +24,6 @@ def has_environment_marker_platform_impl_support():
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
     'parsel>=1.5.0',
-    'PyDispatcher>=2.0.5',
     'pyOpenSSL>=16.2.0',
     'queuelib>=1.4.2',
     'service_identity>=16.0.0',
@@ -34,11 +33,12 @@ def has_environment_marker_platform_impl_support():
     'itemadapter>=0.1.0',
 ]
 extras_require = {}
-
+cpython_dependencies = [
+    'lxml>=3.5.0',
+    'PyDispatcher>=2.0.5',
+]
 if has_environment_marker_platform_impl_support():
-    extras_require[':platform_python_implementation == "CPython"'] = [
-        'lxml>=3.5.0',
-    ]
+    extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
     extras_require[':platform_python_implementation == "PyPy"'] = [
         # Earlier lxml versions are affected by
         # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
@@ -49,7 +49,7 @@ def has_environment_marker_platform_impl_support():
         'PyPyDispatcher>=2.1.0',
     ]
 else:
-    install_requires.append('lxml>=3.5.0')
+    install_requires.extend(cpython_dependencies)
 
 
 setup(

tests/requirements-py3.txt

@@ -1,8 +1,6 @@
 # Tests requirements
 attrs
 dataclasses; python_version == '3.6'
-mitmproxy; python_version >= '3.7'
-mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7'
 pyftpdlib
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1

tests/test_dependencies.py

@@ -1,8 +1,14 @@
+import os
+import re
+from configparser import ConfigParser
 from importlib import import_module
+
+from twisted import version as twisted_version
 from twisted.trial import unittest
 
 
 class ScrapyUtilsTest(unittest.TestCase):
+
     def test_required_openssl_version(self):
         try:
             module = import_module('OpenSSL')
@@ -13,6 +19,32 @@ def test_required_openssl_version(self):
             installed_version = [int(x) for x in module.__version__.split('.')[:2]]
             assert installed_version >= [0, 6], "OpenSSL >= 0.6 required"
 
+    def test_pinned_twisted_version(self):
+        """When running tests within a Tox environment with pinned
+        dependencies, make sure that the version of Twisted is the pinned
+        version.
+
+        See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
+        """
+        if not os.environ.get('_SCRAPY_PINNED', None):
+            self.skipTest('Not in a pinned environment')
+
+        tox_config_file_path = os.path.join(
+            os.path.dirname(__file__),
+            '..',
+            'tox.ini',
+        )
+        config_parser = ConfigParser()
+        config_parser.read(tox_config_file_path)
+        pattern = r'Twisted==([\d.]+)'
+        match = re.search(pattern, config_parser['pinned']['deps'])
+        pinned_twisted_version_string = match[1]
+
+        self.assertEqual(
+            twisted_version.short(),
+            pinned_twisted_version_string
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_downloadermiddleware.py

@@ -1,11 +1,13 @@
 import asyncio
-from unittest import mock
+from unittest import mock, SkipTest
 
 from pytest import mark
+from twisted import version as twisted_version
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
+from twisted.python.versions import Version
 
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
@@ -211,10 +213,21 @@ def process_request(self, request, spider):
         self.assertFalse(download_func.called)
 
 
+@mark.usefixtures('reactor_pytest')
 class MiddlewareUsingCoro(ManagerTestCase):
     """Middlewares using asyncio coroutines should work"""
 
     def test_asyncdef(self):
+        if (
+            self.reactor_pytest == 'asyncio'
+            and twisted_version < Version('twisted', 18, 4, 0)
+        ):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'hangs when using AsyncIO and Twisted versions lower than '
+                '18.4.0'
+            )
+
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:
@@ -235,6 +248,12 @@ async def process_request(self, request, spider):
 
     @mark.only_asyncio()
     def test_asyncdef_asyncio(self):
+        if twisted_version < Version('twisted', 18, 4, 0):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'hangs when using Twisted versions lower than 18.4.0'
+            )
+
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware: