Travis CI → GitHub Actions

.github/workflows/checks.yml

@@ -0,0 +1,27 @@
+name: Checks
+on: [push, pull_request]
+
+jobs:
+  checks:
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - python-version: 3.8  # Keep in sync with .readthedocs.yml
+          env:
+            TOXENV: docs
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run check
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox
+        tox

.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish
+on: [push]
+
+jobs:
+  publish:
+    runs-on: ubuntu-18.04
+    if: startsWith(github.event.ref, 'refs/tags/')
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Check Tag
+      id: check-release-tag
+      run: |
+        if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
+          echo ::set-output name=release_tag::true
+        fi
+
+    - name: Publish to PyPI
+      if: steps.check-release-tag.outputs.release_tag == 'true'
+      run: |
+        pip install --upgrade setuptools wheel twine
+        python setup.py sdist bdist_wheel
+        export TWINE_USERNAME=__token__
+        export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
+        twine upload dist/*

.github/workflows/tests.yml

@@ -0,0 +1,81 @@
+name: Ubuntu
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - python-version: 2.7
+          env:
+            TOXENV: py27
+        - python-version: 2.7
+          env:
+            TOXENV: py27-pinned
+        - python-version: 2.7
+          env:
+            TOXENV: py27-extra-deps
+        - python-version: pypy3
+          env:
+            TOXENV: pypy
+            PYPY_VERSION: '2.7-v7.3.5'
+        - python-version: pypy3
+          env:
+            TOXENV: pypy3
+            PYPY_VERSION: '3.7-v7.3.5'
+        - python-version: 3.5
+          env:
+            TOXENV: py35
+        - python-version: 3.5
+          env:
+            TOXENV: py35-pinned
+        - python-version: 3.6
+          env:
+            TOXENV: py36
+        - python-version: 3.7
+          env:
+            TOXENV: py37
+        - python-version: 3.8
+          env:
+            TOXENV: py38
+        - python-version: 3.8
+          env:
+            TOXENV: py38-extra-deps
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install system libraries
+      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
+      run: |
+        sudo apt-get update
+        # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
+        sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: |
+        if [[ ! -z "$PYPY_VERSION" ]]; then
+          export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+          wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
+          tar -jxf ${PYPY_VERSION}.tar.bz2
+          if [ "${PYPY_VERSION:4:1}" = "2" ]; then
+            pip install -U virtualenv
+            virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
+          else
+            $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
+          fi
+          source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+        fi
+        pip install -U tox
+        tox
+
+    - name: Upload coverage report
+      run: bash <(curl -s https://codecov.io/bash)

.readthedocs.yml

@@ -0,0 +1,16 @@
+version: 2
+formats: all
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+
+build:
+  image: latest
+
+python:
+  # For available versions, see:
+  # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
+  version: 3.8  # Keep in sync with .github/workflows/checks.yml
+  install:
+    - requirements: docs/requirements.txt
+    - path: .

docs/topics/downloader-middleware.rst

@@ -828,7 +828,9 @@ REDIRECT_MAX_TIMES
 
 Default: ``20``
 
-The maximum number of redirections that will be followed for a single request.
+Defines the maximum times a request can be redirected. After this maximum the
+request's response is returned as is. We used Firefox default value for the
+same task.
 
 MetaRefreshMiddleware
 ---------------------

docs/topics/settings.rst

@@ -1104,17 +1104,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
 DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
 this value if you're experiencing problems with insufficient blocking IO.
 
-.. setting:: REDIRECT_MAX_TIMES
-
-REDIRECT_MAX_TIMES
-------------------
-
-Default: ``20``
-
-Defines the maximum times a request can be redirected. After this maximum the
-request's response is returned as is. We used Firefox default value for the
-same task.
-
 .. setting:: REDIRECT_PRIORITY_ADJUST
 
 REDIRECT_PRIORITY_ADJUST
@@ -1396,17 +1385,6 @@ Default: ``True``
 A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
 will be enabled (provided its extension is also enabled).
 
-.. setting:: TELNETCONSOLE_PORT
-
-TELNETCONSOLE_PORT
-------------------
-
-Default: ``[6023, 6073]``
-
-The port range to use for the telnet console. If set to ``None`` or ``0``, a
-dynamically assigned port is used. For more info see
-:ref:`topics-telnetconsole`.
-
 .. setting:: TEMPLATES_DIR
 
 TEMPLATES_DIR

requirements-py2.txt

@@ -3,9 +3,10 @@ PyDispatcher>=2.0.5
 w3lib>=1.17.0
 protego>=0.1.15
 
-pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0  # Earlier versions would fail to install
+pyOpenSSL>=16.2.0       # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2,<1.6.0  # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+                        # Greater versions implement peek support, which breaks tests.
+cryptography>=2.0       # Earlier versions would fail to install
 
 # Reference versions taken from
 # https://packages.ubuntu.com/xenial/python/
@@ -14,5 +15,5 @@ cssselect>=0.9.1
 lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
-Twisted>=16.0.0
-zope.interface>=4.1.3
+Twisted>=16.0.0,<21.2.0  # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020
+zope.interface>=4.1.3,<5  # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489

requirements-py3.txt

@@ -1,12 +1,13 @@
 parsel>=1.5.0
 PyDispatcher>=2.0.5
-Twisted>=17.9.0
+Twisted>=17.9.0,<21.2.0  # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020
 w3lib>=1.17.0
 protego>=0.1.15
 
-pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0  # Earlier versions would fail to install
+pyOpenSSL>=16.2.0       # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2,<1.6.0  # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+                        # Greater versions implement peek support, which breaks tests.
+cryptography>=2.0       # Earlier versions would fail to install
 
 # Reference versions taken from
 # https://packages.ubuntu.com/xenial/python/
@@ -15,4 +16,4 @@ cssselect>=0.9.1
 lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
-zope.interface>=4.1.3
+zope.interface>=4.1.3,<5  # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489

tests/test_utils_python.py

@@ -5,6 +5,7 @@
 from itertools import count
 import platform
 import six
+from sys import version_info
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
@@ -234,10 +235,12 @@ def __call__(self, a, b, c):
             stripself = not six.PY2  # PyPy3 exposes them as methods
             self.assertEqual(
                 get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself), ['list'])
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), stripself), ['obj'])
-
+            if version_info < (3, 6):
+                self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+            else:
+                self.assertEqual(get_func_args(" ".join, stripself), ['iterable'])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])