From 2f740b29678d3e68775191c4d4ca3017ad5ad493 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 15:13:57 -0300 Subject: [PATCH 01/38] Run most CI processes on GH actions --- .github/workflows/main.yml | 72 ++++++++++++++++++++++++++++++++++---- .travis.yml | 26 +------------- 2 files changed, 66 insertions(+), 32 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 28771216c82..4a9952c4741 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,19 +2,77 @@ name: Run test suite on: [push, pull_request] jobs: - test-windows: - name: "Windows Tests" + build: + name: Build runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest] - python-version: [3.7, 3.8] - env: [TOXENV: py] include: + # non-testing jobs + - os: ubuntu-18.04 + python-version: 3.8 + env: + TOXENV: security + - os: ubuntu-18.04 + python-version: 3.8 + env: + TOXENV: flake8 + - os: ubuntu-18.04 + python-version: 3.8 + env: + TOXENV: pylint + - os: ubuntu-18.04 + python-version: 3.8 + env: + TOXENV: typing + - os: ubuntu-18.04 + python-version: 3.7 # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + # tests + - os: ubuntu-18.04 + python-version: 3.8 + env: + TOXENV: py + - os: ubuntu-18.04 + python-version: 3.8 + env: + TOXENV: py + + # windows tests - os: windows-latest - python-version: 3.6 + python-version: 3.7 + env: + TOXENV: py + - os: windows-latest + python-version: 3.8 + env: + TOXENV: py + + # pinned deps + - os: ubuntu-18.04 + python-version: 3.6.1 + env: + TOXENV: pinned + - os: ubuntu-18.04 + python-version: 3.6.1 + env: + TOXENV: asyncio-pinned + # - os: windows-latest + # python-version: 3.6 + # env: + # TOXENV: windows-pinned + + # extras + - os: ubuntu-18.04 + python-version: 3.8 + env: + TOXENV: extra-deps + - os: ubuntu-18.04 + python-version: 3.8 env: - TOXENV: windows-pinned + TOXENV: asyncio steps: - uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml index b883c5b78d3..a973fcd8c85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,39 +7,15 @@ branches: - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: include: - - env: TOXENV=security - python: 3.8 - - env: TOXENV=flake8 - python: 3.8 - - env: TOXENV=pylint - python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml - - env: TOXENV=typing - python: 3.8 - - env: TOXENV=pinned - python: 3.6.1 - - env: TOXENV=asyncio-pinned - python: 3.6.1 - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - - env: TOXENV=py - python: 3.6 - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 - - env: TOXENV=py - python: 3.7 - - env: TOXENV=py PYPI_RELEASE_JOB=true python: 3.8 dist: bionic - - env: TOXENV=extra-deps - python: 3.8 - dist: bionic - - env: TOXENV=asyncio - python: 3.8 - dist: bionic + install: - | if [[ ! -z "$PYPY_VERSION" ]]; then From b7337e91433acdeafa46e4b7f3399777b4f407e0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 15:16:51 -0300 Subject: [PATCH 02/38] CI: Pin python version --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4a9952c4741..15294067192 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,11 +52,11 @@ jobs: # pinned deps - os: ubuntu-18.04 - python-version: 3.6.1 + python-version: 3.6.12 env: TOXENV: pinned - os: ubuntu-18.04 - python-version: 3.6.1 + python-version: 3.6.12 env: TOXENV: asyncio-pinned # - os: windows-latest From ccd78a710b7aff0c88f1d3b72b63bdad60cc90e4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 15:26:20 -0300 Subject: [PATCH 03/38] Update CI --- .github/workflows/main.yml | 20 ++++++++++++-------- .travis.yml | 2 -- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 15294067192..10512ecb09a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,6 +39,10 @@ jobs: python-version: 3.8 env: TOXENV: py + - os: ubuntu-18.04 + python-version: pypy3 + env: + TOXENV: pypy3 # windows tests - os: windows-latest @@ -51,14 +55,14 @@ jobs: TOXENV: py # pinned deps - - os: ubuntu-18.04 - python-version: 3.6.12 - env: - TOXENV: pinned - - os: ubuntu-18.04 - python-version: 3.6.12 - env: - TOXENV: asyncio-pinned + # - os: ubuntu-18.04 + # python-version: 3.6.12 + # env: + # TOXENV: pinned + # - os: ubuntu-18.04 + # python-version: 3.6.12 + # env: + # TOXENV: asyncio-pinned # - os: windows-latest # python-version: 3.6 # env: diff --git a/.travis.yml b/.travis.yml index a973fcd8c85..4b1415904f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,6 @@ matrix: - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 - - env: TOXENV=py PYPI_RELEASE_JOB=true python: 3.8 dist: bionic From e8851d1ea313f65125e719890d527fc01e0f5481 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 15:38:56 -0300 Subject: [PATCH 04/38] Update CI --- .github/workflows/main.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 10512ecb09a..2f563eeed2f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: Run test suite +name: Build on: [push, pull_request] jobs: @@ -82,12 +82,16 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Run test suite + - name: Install libraries + if: ${{ env.TOXENV == 'pypy3' }} + run: apt-get update && apt-get install libxml2-dev libxslt-dev + + - name: Run check env: ${{ matrix.env }} run: | - pip install -U tox twine wheel codecov + pip install -U tox tox From 9530784a40e430ba8f6597ccd0c56716114c6a61 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 15:50:40 -0300 Subject: [PATCH 05/38] Update CI --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2f563eeed2f..8bd68cb10c7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,7 +87,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install libraries - if: ${{ env.TOXENV == 'pypy3' }} + if: env.TOXENV == 'pypy3' run: apt-get update && apt-get install libxml2-dev libxslt-dev - name: Run check From 1aba857f66caca7896c94f3c1e447dc2e287edb0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 15:58:06 -0300 Subject: [PATCH 06/38] Update CI --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8bd68cb10c7..ca9e71f6810 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,7 +87,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install libraries - if: env.TOXENV == 'pypy3' + if: matrix.env.TOXENV == 'pypy3' run: apt-get update && apt-get install libxml2-dev libxslt-dev - name: Run check From 6b3e1c34f9ce730e9bdb4e3f96bc00a8e7d8866a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 16:01:03 -0300 Subject: [PATCH 07/38] Update CI --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca9e71f6810..2264e72aa38 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,7 +88,9 @@ jobs: - name: Install libraries if: matrix.env.TOXENV == 'pypy3' - run: apt-get update && apt-get install libxml2-dev libxslt-dev + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxslt-dev - name: Run check env: ${{ matrix.env }} From b6fe5c361a934565a73db7026bfe237348a3453b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 17:08:37 -0300 Subject: [PATCH 08/38] Download specific pypy version --- .github/workflows/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2264e72aa38..bcbec347e50 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,7 @@ jobs: python-version: pypy3 env: TOXENV: pypy3 + PYPY_VERSION: 3.6-v7.3.1 # windows tests - os: windows-latest @@ -67,6 +68,11 @@ jobs: # python-version: 3.6 # env: # TOXENV: windows-pinned + # - os: ubuntu-18.04 + # python-version: pypy3 + # env: + # TOXENV: pypy3-pinned + # PYPY_VERSION: 3.6-v7.2.0 # extras - os: ubuntu-18.04 @@ -92,6 +98,16 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Download pypy + if: matrix.env.TOXENV == 'pypy3' || matrix.env.TOXENV == 'pypy3-pinned' + env: ${{ matrix.env }} + run: | + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + - name: Run check env: ${{ matrix.env }} run: | From c0bc1670cabe36124a34d08c3fce1931ae18fd29 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 17:14:47 -0300 Subject: [PATCH 09/38] Virtualenv creation --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bcbec347e50..4a24e5e620f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -105,7 +105,7 @@ jobs: export PYPY_VERSION="pypy$PYPY_VERSION-linux64" wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - name: Run check From 937c510a676d91efff67a6cdf139127a214f010b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 21:16:23 -0300 Subject: [PATCH 10/38] pypy3 adjustments --- .github/workflows/main.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4a24e5e620f..f5ffa00293f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,23 +93,20 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install libraries - if: matrix.env.TOXENV == 'pypy3' + if: matrix.python-version == 'pypy3' run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev - - name: Download pypy - if: matrix.env.TOXENV == 'pypy3' || matrix.env.TOXENV == 'pypy3-pinned' - env: ${{ matrix.env }} - run: | - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - - name: Run check env: ${{ matrix.env }} run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi pip install -U tox tox From 2513db9aa6627f723660859cc7465f301f24aa2e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 21:24:50 -0300 Subject: [PATCH 11/38] pypy3 adjustments --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f5ffa00293f..4d0fb339b78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -101,7 +101,7 @@ jobs: - name: Run check env: ${{ matrix.env }} run: | - if [[ ! -z "$PYPY_VERSION" ]]; then + if ([[ ! -z "$PYPY_VERSION" ]]); then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 From 29769deecc83d73d21b3c7398308864772540534 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 11 Dec 2020 21:34:28 -0300 Subject: [PATCH 12/38] pypy3 adjustments --- .github/workflows/main.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4d0fb339b78..896d0a1791a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,10 +98,18 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev - - name: Run check + - name: Run check (windows) + if: startsWith(matrix.os, 'windows') env: ${{ matrix.env }} run: | - if ([[ ! -z "$PYPY_VERSION" ]]); then + pip install -U tox + tox + + - name: Run check (ubuntu) + if: startsWith(matrix.os, 'ubuntu') + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 From 7b778e33d4a4dd016d2c5cfd86496c86e2d17cff Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 10 Dec 2020 13:14:06 -0300 Subject: [PATCH 13/38] Update depencencies --- setup.py | 4 ++-- tests/requirements-py3.txt | 12 +++++++----- tox.ini | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 35736b75f11..3956e802076 100644 --- a/setup.py +++ b/setup.py @@ -20,12 +20,12 @@ def has_environment_marker_platform_impl_support(): install_requires = [ 'Twisted>=17.9.0', - 'cryptography>=2.0', + 'cryptography>=2.1.4', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', - 'pyOpenSSL>=16.2.0', + 'pyOpenSSL>=17.5.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', 'w3lib>=1.17.0', diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 7f8a5c52e93..f22ce90f397 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,8 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.7' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' +mitmproxy >= 4.0.4; python_version >= '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 @@ -14,8 +14,10 @@ testfixtures uvloop; platform_system != "Windows" # optional for shell wrapper tests -bpython -brotlipy # optional for HTTP compress downloader middleware tests -zstandard # optional for HTTP compress downloader middleware tests -ipython +bpython >= 0.20.1 +brotlipy >= 0.7.0 # optional for HTTP compress downloader middleware tests +zstandard >= 0.14.1 # optional for HTTP compress downloader middleware tests +jedi >= 0.17.2 # from ipython +ipython >= 7.16.1; python_version == '3.6' +ipython >= 7.19.0; python_version >= '3.7' pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index ea356c56a3f..33a7388d5de 100644 --- a/tox.ini +++ b/tox.ini @@ -61,13 +61,13 @@ commands = [pinned] deps = -ctests/constraints.txt - cryptography==2.0 + cryptography==2.1.4 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 PyDispatcher==2.0.5 - pyOpenSSL==16.2.0 + pyOpenSSL==17.5.0 queuelib==1.4.2 service_identity==16.0.0 Twisted==17.9.0 From a5088c4e2094e3e4c59a5f60141e6bf1a1cfa938 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 14 Dec 2020 14:50:23 -0300 Subject: [PATCH 14/38] Enable test envs with pinned dependencies --- .github/workflows/main.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 896d0a1791a..9e68c8bf262 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,23 +56,23 @@ jobs: TOXENV: py # pinned deps - # - os: ubuntu-18.04 - # python-version: 3.6.12 - # env: - # TOXENV: pinned - # - os: ubuntu-18.04 - # python-version: 3.6.12 - # env: - # TOXENV: asyncio-pinned - # - os: windows-latest - # python-version: 3.6 - # env: - # TOXENV: windows-pinned - # - os: ubuntu-18.04 - # python-version: pypy3 - # env: - # TOXENV: pypy3-pinned - # PYPY_VERSION: 3.6-v7.2.0 + - os: ubuntu-18.04 + python-version: 3.6.12 + env: + TOXENV: pinned + - os: ubuntu-18.04 + python-version: 3.6.12 + env: + TOXENV: asyncio-pinned + - os: windows-latest + python-version: 3.6 + env: + TOXENV: windows-pinned + - os: ubuntu-18.04 + python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.6-v7.2.0 # extras - os: ubuntu-18.04 From 7045d2c32e3e9acce2281d9c573af8db4f6c6e8c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 14 Dec 2020 14:55:09 -0300 Subject: [PATCH 15/38] Install system libraries for pinned envs --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9e68c8bf262..99ada824bf4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,8 +92,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install libraries - if: matrix.python-version == 'pypy3' + - name: Install system libraries + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev From 31437915693ea85648b49dc2094bccd0eb420faa Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 14 Dec 2020 15:02:30 -0300 Subject: [PATCH 16/38] Install system libraries only for ubuntu-based envs --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 99ada824bf4..6cbbb6a88f8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,7 +93,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + if: startsWith(matrix.os, 'ubuntu') && (matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')) run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev From 22fce67a7e2893766bfad0ea2fec1317c90037ee Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Dec 2020 13:07:52 -0300 Subject: [PATCH 17/38] Remove travis config file --- .github/workflows/main.yml | 52 ++++++++++++++++++++++++++++++-------- .travis.yml | 49 ----------------------------------- 2 files changed, 41 insertions(+), 60 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6cbbb6a88f8..fe194e4fa82 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,18 +32,18 @@ jobs: # tests - os: ubuntu-18.04 - python-version: 3.8 + python-version: 3.7 env: TOXENV: py - os: ubuntu-18.04 python-version: 3.8 env: TOXENV: py - - os: ubuntu-18.04 - python-version: pypy3 - env: - TOXENV: pypy3 - PYPY_VERSION: 3.6-v7.3.1 + # - os: ubuntu-18.04 + # python-version: pypy3 + # env: + # TOXENV: pypy3 + # PYPY_VERSION: 3.6-v7.3.1 # windows tests - os: windows-latest @@ -68,11 +68,11 @@ jobs: python-version: 3.6 env: TOXENV: windows-pinned - - os: ubuntu-18.04 - python-version: pypy3 - env: - TOXENV: pypy3-pinned - PYPY_VERSION: 3.6-v7.2.0 + # - os: ubuntu-18.04 + # python-version: pypy3 + # env: + # TOXENV: pypy3-pinned + # PYPY_VERSION: 3.6-v7.2.0 # extras - os: ubuntu-18.04 @@ -118,3 +118,33 @@ jobs: fi pip install -U tox tox + + publish: + name: "Publish package to PyPI" + runs-on: ubuntu-18.04 + needs: [build] + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4b1415904f0..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,49 +0,0 @@ -language: python -dist: xenial -branches: - only: - - master - - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -matrix: - include: - - - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - - - env: TOXENV=py PYPI_RELEASE_JOB=true - python: 3.8 - dist: bionic - -install: - - | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov - -script: tox -after_success: - - codecov -notifications: - irc: - use_notice: true - skip_join: true - channels: - - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: scrapy - password: - secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA= - on: - tags: true - repo: scrapy/scrapy - condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" From abeb7dd092a54cf63506f6cad18f3bcf9e4f1943 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Dec 2020 16:32:50 -0300 Subject: [PATCH 18/38] Restore minimum pyOpenSSL version --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d1894149baf..cce848cc337 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def has_environment_marker_platform_impl_support(): 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'pyOpenSSL>=17.5.0', + 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', 'w3lib>=1.17.0', diff --git a/tox.ini b/tox.ini index 65fac2e4ee3..92c71db6593 100644 --- a/tox.ini +++ b/tox.ini @@ -72,7 +72,7 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==17.5.0 + pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 Twisted==17.9.0 From b261f52ebfad9f1b99e0c2f9ab9b94494259d1d3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Dec 2020 16:54:18 -0300 Subject: [PATCH 19/38] Restore minimum cryptography version --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cce848cc337..b5c42a3c267 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ 'Twisted>=17.9.0', - 'cryptography>=2.1.4', + 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', diff --git a/tox.ini b/tox.ini index 92c71db6593..1a9d5b5c300 100644 --- a/tox.ini +++ b/tox.ini @@ -67,7 +67,7 @@ commands = [pinned] deps = -ctests/constraints.txt - cryptography==2.1.4 + cryptography==2.0 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From f0dbc08e37835b6d961d7b525ef70adb1b2d2f1b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Dec 2020 17:19:30 -0300 Subject: [PATCH 20/38] Restore test requirements --- tests/requirements-py3.txt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index c9040fd366e..7ee99292dde 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -12,10 +12,7 @@ testfixtures uvloop; platform_system != "Windows" # optional for shell wrapper tests -bpython >= 0.20.1 -brotlipy >= 0.7.0 # optional for HTTP compress downloader middleware tests -zstandard >= 0.14.1 # optional for HTTP compress downloader middleware tests -jedi >= 0.17.2 # from ipython -ipython >= 7.16.1; python_version == '3.6' -ipython >= 7.19.0; python_version >= '3.7' -pywin32; sys_platform == "win32" +bpython +brotlipy # optional for HTTP compress downloader middleware tests +zstandard # optional for HTTP compress downloader middleware tests +ipython From 5909d89d8c00235544a53d52bc7940448f5a1913 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Dec 2020 17:25:48 -0300 Subject: [PATCH 21/38] Restore pywin32 requirement --- tests/requirements-py3.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 7ee99292dde..68b856a88d3 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -16,3 +16,4 @@ bpython brotlipy # optional for HTTP compress downloader middleware tests zstandard # optional for HTTP compress downloader middleware tests ipython +pywin32; sys_platform == "win32" From 3cb1dc63ebd7b1a8c9ad87ffad2e2c64f73c6ca2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 16 Dec 2020 18:02:47 -0300 Subject: [PATCH 22/38] Disable test under pypy --- tests/test_webclient.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a60181a3a89..f935a86892b 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,7 +4,10 @@ """ import os import shutil +import sys +from pkg_resources import parse_version +import cryptography import OpenSSL.SSL from twisted.trial import unittest from twisted.web import server, static, util, resource @@ -414,6 +417,8 @@ def testPayload(self): ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): + if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): + self.skipTest("This does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) From fc5070ffda9b9653c671892ba1e67d0901b9aae2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 16 Dec 2020 18:16:48 -0300 Subject: [PATCH 23/38] Upload coverage report from CI --- .github/workflows/main.yml | 8 ++++++++ tox.ini | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6379d1b1b12..ce31ed441d0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -106,6 +106,7 @@ jobs: tox - name: Run check (ubuntu) + id: ubuntu-tests if: startsWith(matrix.os, 'ubuntu') env: ${{ matrix.env }} run: | @@ -118,6 +119,13 @@ jobs: fi pip install -U tox tox + if [ -f "coverage.xml" ]; then + echo ::set-output name=upload_coverage_report::true + fi + + - name: Upload coverage report + if: steps.ubuntu-tests.outputs.upload_coverage_report == 'true' + run: bash <(curl -s https://codecov.io/bash) publish: name: "Publish package to PyPI" diff --git a/tox.ini b/tox.ini index 1a9d5b5c300..aef196a937d 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} + py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} [testenv:typing] basepython = python3 From eb961acb5ee440cbd3d7e3e7faa8405f5b2dc9d2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 16 Dec 2020 18:45:07 -0300 Subject: [PATCH 24/38] Remove install_command from tox.ini --- tox.ini | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tox.ini b/tox.ini index aef196a937d..e70aef2d291 100644 --- a/tox.ini +++ b/tox.ini @@ -87,10 +87,6 @@ deps = botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 -install_command = - # --use-feature=2020-resolver is required, otherwise the latest verion of - # Twisted gets installed. - pip install --use-feature=2020-resolver {opts} {packages} setenv = _SCRAPY_PINNED=true @@ -99,8 +95,6 @@ deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 -install_command = - {[pinned]install_command} setenv = {[pinned]setenv} @@ -112,8 +106,6 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 -install_command = - {[pinned]install_command} setenv = {[pinned]setenv} @@ -122,10 +114,6 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser -install_command = - # Test --use-feature=2020-resolver for the latest version of all - # dependencies. - pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = @@ -133,8 +121,6 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} -install_command = - {[pinned]install_command} commands = {[testenv:asyncio]commands} setenv = {[pinned]setenv} @@ -150,8 +136,6 @@ deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 -install_command = - {[pinned]install_command} commands = {[testenv:pypy3]commands} setenv = {[pinned]setenv} From af4c3961a361a83f114921a4e6419c56dce38cb5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 24 Dec 2020 12:36:24 -0300 Subject: [PATCH 25/38] Update build badge on README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a8f2ba52b64..b9aecd99f23 100644 --- a/README.rst +++ b/README.rst @@ -10,8 +10,8 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: https://travis-ci.org/scrapy/scrapy +.. image:: https://github.com/scrapy/scrapy/workflows/Build/badge.svg + :target: https://github.com/scrapy/scrapy/actions :alt: Build Status .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg From f8d6b010ead6c6f06547561c381a336c79aa7f03 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Dec 2020 02:19:36 -0300 Subject: [PATCH 26/38] Split CI config files --- .github/workflows/checks.yml | 39 +++++++ .github/workflows/foo.yml | 20 ++++ .github/workflows/main.yml | 158 ---------------------------- .github/workflows/publish.yml | 33 ++++++ .github/workflows/tests-ubuntu.yml | 70 ++++++++++++ .github/workflows/tests-windows.yml | 33 ++++++ 6 files changed, 195 insertions(+), 158 deletions(-) create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/foo.yml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/tests-ubuntu.yml create mode 100644 .github/workflows/tests-windows.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 00000000000..098d3bae191 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,39 @@ +name: Checks +on: [push, pull_request] + +jobs: + checks: + name: Checks + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.8 + env: + TOXENV: security + - python-version: 3.8 + env: + TOXENV: flake8 + - python-version: 3.8 + env: + TOXENV: pylint + - python-version: 3.8 + env: + TOXENV: typing + - python-version: 3.7 # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run check + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.github/workflows/foo.yml b/.github/workflows/foo.yml new file mode 100644 index 00000000000..8ac6b9013b5 --- /dev/null +++ b/.github/workflows/foo.yml @@ -0,0 +1,20 @@ +name: Say hello +on: [push] + +jobs: + say-hello: + name: Say hello + runs-on: ubuntu-18.04 + needs: [checks, tests-ubuntu, tests-windows] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Say hello + run: | + echo "Hello! This should work only if tests and checks succeded" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index ce31ed441d0..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,158 +0,0 @@ -name: Build -on: [push, pull_request] - -jobs: - build: - name: Build - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - # non-testing jobs - - os: ubuntu-18.04 - python-version: 3.8 - env: - TOXENV: security - - os: ubuntu-18.04 - python-version: 3.8 - env: - TOXENV: flake8 - - os: ubuntu-18.04 - python-version: 3.8 - env: - TOXENV: pylint - - os: ubuntu-18.04 - python-version: 3.8 - env: - TOXENV: typing - - os: ubuntu-18.04 - python-version: 3.7 # Keep in sync with .readthedocs.yml - env: - TOXENV: docs - - # tests - - os: ubuntu-18.04 - python-version: 3.7 - env: - TOXENV: py - - os: ubuntu-18.04 - python-version: 3.8 - env: - TOXENV: py - - os: ubuntu-18.04 - python-version: pypy3 - env: - TOXENV: pypy3 - PYPY_VERSION: 3.6-v7.3.1 - - # windows tests - - os: windows-latest - python-version: 3.7 - env: - TOXENV: py - - os: windows-latest - python-version: 3.8 - env: - TOXENV: py - - # pinned deps - - os: ubuntu-18.04 - python-version: 3.6.12 - env: - TOXENV: pinned - - os: ubuntu-18.04 - python-version: 3.6.12 - env: - TOXENV: asyncio-pinned - - os: windows-latest - python-version: 3.6 - env: - TOXENV: windows-pinned - - os: ubuntu-18.04 - python-version: pypy3 - env: - TOXENV: pypy3-pinned - PYPY_VERSION: 3.6-v7.2.0 - - # extras - - os: ubuntu-18.04 - python-version: 3.8 - env: - TOXENV: extra-deps - - os: ubuntu-18.04 - python-version: 3.8 - env: - TOXENV: asyncio - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system libraries - if: startsWith(matrix.os, 'ubuntu') && (matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')) - run: | - sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev - - - name: Run check (windows) - if: startsWith(matrix.os, 'windows') - env: ${{ matrix.env }} - run: | - pip install -U tox - tox - - - name: Run check (ubuntu) - id: ubuntu-tests - if: startsWith(matrix.os, 'ubuntu') - env: ${{ matrix.env }} - run: | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - pip install -U tox - tox - if [ -f "coverage.xml" ]; then - echo ::set-output name=upload_coverage_report::true - fi - - - name: Upload coverage report - if: steps.ubuntu-tests.outputs.upload_coverage_report == 'true' - run: bash <(curl -s https://codecov.io/bash) - - publish: - name: "Publish package to PyPI" - runs-on: ubuntu-18.04 - needs: [build] - if: startsWith(github.event.ref, 'refs/tags/') - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Check Tag - id: check-release-tag - run: | - if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then - echo ::set-output name=release_tag::true - fi - - - name: Publish to PyPI - if: steps.check-release-tag.outputs.release_tag == 'true' - run: | - pip install --upgrade setuptools wheel twine - python setup.py sdist bdist_wheel - export TWINE_USERNAME=__token__ - export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} - twine upload dist/* diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000000..327d655524a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,33 @@ +name: Publish +on: [push] + +jobs: + publish: + name: Publish package to PyPI + runs-on: ubuntu-18.04 + needs: [checks, tests-ubuntu, tests-windows] + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml new file mode 100644 index 00000000000..16db07697b4 --- /dev/null +++ b/.github/workflows/tests-ubuntu.yml @@ -0,0 +1,70 @@ +name: Tests - Ubuntu +on: [push, pull_request] + +jobs: + tests-ubuntu: + name: Tests - Ubuntu + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: 3.6-v7.3.1 + + # pinned deps + - python-version: 3.6.12 + env: + TOXENV: pinned + - python-version: 3.6.12 + env: + TOXENV: asyncio-pinned + - python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.6-v7.2.0 + + # extras + - python-version: 3.8 + env: + TOXENV: extra-deps + - python-version: 3.8 + env: + TOXENV: asyncio + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system libraries + if: matrix.python-version == 'pypy3' + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxslt-dev + + - name: Run tests + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + pip install -U tox + tox + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml new file mode 100644 index 00000000000..7f1534ca1c1 --- /dev/null +++ b/.github/workflows/tests-windows.yml @@ -0,0 +1,33 @@ +name: Tests - Windows +on: [push, pull_request] + +jobs: + tests-windows: + name: Tests - Windows + runs-on: windows-latest + strategy: + matrix: + include: + - python-version: 3.6 + env: + TOXENV: windows-pinned + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + env: ${{ matrix.env }} + run: | + pip install -U tox + tox From ac3e6f523297a3aa4d8bb88833e113c4343cbd12 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Dec 2020 02:22:59 -0300 Subject: [PATCH 27/38] Install system libraries in pinned env --- .github/workflows/foo.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/foo.yml b/.github/workflows/foo.yml index 8ac6b9013b5..d94e3749045 100644 --- a/.github/workflows/foo.yml +++ b/.github/workflows/foo.yml @@ -1,5 +1,5 @@ name: Say hello -on: [push] +on: [push, pull_request] jobs: say-hello: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 16db07697b4..09b75918591 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -48,7 +48,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned' run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev From abd1092a175ec78c7542c536cb5b0504937ff5c4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Dec 2020 02:24:24 -0300 Subject: [PATCH 28/38] Fix typo --- .github/workflows/tests-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 09b75918591..e2a8dcacad7 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -48,7 +48,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned' + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev From 5458a8fe09efa3427afe60e54c311c58c7256df1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Dec 2020 02:38:17 -0300 Subject: [PATCH 29/38] Publish CI job does not depend on previous jobs --- .github/workflows/foo.yml | 20 -------------------- .github/workflows/publish.yml | 1 - 2 files changed, 21 deletions(-) delete mode 100644 .github/workflows/foo.yml diff --git a/.github/workflows/foo.yml b/.github/workflows/foo.yml deleted file mode 100644 index d94e3749045..00000000000 --- a/.github/workflows/foo.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Say hello -on: [push, pull_request] - -jobs: - say-hello: - name: Say hello - runs-on: ubuntu-18.04 - needs: [checks, tests-ubuntu, tests-windows] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Say hello - run: | - echo "Hello! This should work only if tests and checks succeded" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 327d655524a..d98ec711192 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,7 +5,6 @@ jobs: publish: name: Publish package to PyPI runs-on: ubuntu-18.04 - needs: [checks, tests-ubuntu, tests-windows] if: startsWith(github.event.ref, 'refs/tags/') steps: From c44a61dae92c9a58fc4d3b94a7be2aca09f01e78 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Dec 2020 11:36:56 -0300 Subject: [PATCH 30/38] Remove job names --- .github/workflows/checks.yml | 1 - .github/workflows/publish.yml | 1 - .github/workflows/tests-ubuntu.yml | 1 - .github/workflows/tests-windows.yml | 1 - 4 files changed, 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 098d3bae191..2748bf5febc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,7 +3,6 @@ on: [push, pull_request] jobs: checks: - name: Checks runs-on: ubuntu-18.04 strategy: matrix: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d98ec711192..aec6b869669 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,7 +3,6 @@ on: [push] jobs: publish: - name: Publish package to PyPI runs-on: ubuntu-18.04 if: startsWith(github.event.ref, 'refs/tags/') diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index e2a8dcacad7..96687ea6a7a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -3,7 +3,6 @@ on: [push, pull_request] jobs: tests-ubuntu: - name: Tests - Ubuntu runs-on: ubuntu-18.04 strategy: matrix: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 7f1534ca1c1..543c323ede3 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -3,7 +3,6 @@ on: [push, pull_request] jobs: tests-windows: - name: Tests - Windows runs-on: windows-latest strategy: matrix: From 5d61e6bd53c2039f91487b320ba3e4420d655832 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Dec 2020 11:40:56 -0300 Subject: [PATCH 31/38] Remove workflow names --- .github/workflows/checks.yml | 1 - .github/workflows/publish.yml | 1 - .github/workflows/tests-ubuntu.yml | 1 - .github/workflows/tests-windows.yml | 1 - 4 files changed, 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2748bf5febc..c1409916389 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,4 +1,3 @@ -name: Checks on: [push, pull_request] jobs: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aec6b869669..39fac934dfe 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,3 @@ -name: Publish on: [push] jobs: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 96687ea6a7a..056ad3af87e 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,4 +1,3 @@ -name: Tests - Ubuntu on: [push, pull_request] jobs: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 543c323ede3..45bb1d5dd2c 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,4 +1,3 @@ -name: Tests - Windows on: [push, pull_request] jobs: From 215ccdedd2e71e7980075a3e51279451306cd70a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Dec 2020 11:43:11 -0300 Subject: [PATCH 32/38] Restore workflow names --- .github/workflows/checks.yml | 1 + .github/workflows/publish.yml | 1 + .github/workflows/tests-ubuntu.yml | 1 + .github/workflows/tests-windows.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c1409916389..2748bf5febc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,3 +1,4 @@ +name: Checks on: [push, pull_request] jobs: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 39fac934dfe..aec6b869669 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,3 +1,4 @@ +name: Publish on: [push] jobs: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 056ad3af87e..d37070fc16a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,3 +1,4 @@ +name: Tests on: [push, pull_request] jobs: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 45bb1d5dd2c..c0d2aed564f 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,3 +1,4 @@ +name: Tests on: [push, pull_request] jobs: From e1420ea112370e087707b57a9c18fed47d017351 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Dec 2020 11:49:18 -0300 Subject: [PATCH 33/38] Update workflow names --- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index d37070fc16a..f90ee56d958 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,4 +1,4 @@ -name: Tests +name: Ubuntu Tests on: [push, pull_request] jobs: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c0d2aed564f..6dafa498ed3 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,4 +1,4 @@ -name: Tests +name: Windows Tests on: [push, pull_request] jobs: From 51f18d66c048de1d0b835b69a7ad0c0e0b4a9931 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 30 Dec 2020 11:10:54 -0300 Subject: [PATCH 34/38] Update CI badges on Readme --- README.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index b9aecd99f23..d94bef72eb7 100644 --- a/README.rst +++ b/README.rst @@ -10,9 +10,13 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Build/badge.svg - :target: https://github.com/scrapy/scrapy/actions - :alt: Build Status +.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu%20Tests/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3A%22Ubuntu+Tests%22 + :alt: Ubuntu Tests + +.. image:: https://github.com/scrapy/scrapy/workflows/Windows%20Tests/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3A%22Windows+Tests%22 + :alt: Windows Tests .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy From 26a5624afb5b5f64ded55cc72d280f36eb440308 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 30 Dec 2020 11:34:13 -0300 Subject: [PATCH 35/38] Do not install zstandard on pypy --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 68b856a88d3..a86c4ae4f96 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -14,6 +14,6 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython brotlipy # optional for HTTP compress downloader middleware tests -zstandard # optional for HTTP compress downloader middleware tests +zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" From 0f8de903d8b8ba2313633ac4992ade7aa78c5813 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 30 Dec 2020 15:09:12 -0300 Subject: [PATCH 36/38] Add macOS build to CI --- .github/workflows/tests-macos.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/tests-macos.yml diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml new file mode 100644 index 00000000000..dc6fbd96133 --- /dev/null +++ b/.github/workflows/tests-macos.yml @@ -0,0 +1,25 @@ +name: macOS Tests +on: [push, pull_request] + +jobs: + tests-macos: + runs-on: macos-10.15 + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + pip install -U tox + tox -e py + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) From 9eb2b178af1bfbc92aa045c4914270d9ff000e2c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 30 Dec 2020 15:21:27 -0300 Subject: [PATCH 37/38] Shorter names for CI jobs --- .github/workflows/tests-macos.yml | 4 ++-- .github/workflows/tests-ubuntu.yml | 4 ++-- .github/workflows/tests-windows.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index dc6fbd96133..51d27c4050d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,8 +1,8 @@ -name: macOS Tests +name: macOS on: [push, pull_request] jobs: - tests-macos: + tests: runs-on: macos-10.15 strategy: matrix: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f90ee56d958..89c0334e2eb 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,8 +1,8 @@ -name: Ubuntu Tests +name: Ubuntu on: [push, pull_request] jobs: - tests-ubuntu: + tests: runs-on: ubuntu-18.04 strategy: matrix: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 6dafa498ed3..ed2e4075d57 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,8 +1,8 @@ -name: Windows Tests +name: Windows on: [push, pull_request] jobs: - tests-windows: + tests: runs-on: windows-latest strategy: matrix: From 988d7337153343d57c58d6e36e59daf694a27c87 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 31 Dec 2020 10:40:19 -0300 Subject: [PATCH 38/38] Update Readme badges --- README.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index d94bef72eb7..9418d270f39 100644 --- a/README.rst +++ b/README.rst @@ -10,13 +10,17 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu%20Tests/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3A%22Ubuntu+Tests%22 - :alt: Ubuntu Tests +.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu + :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/Windows%20Tests/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3A%22Windows+Tests%22 - :alt: Windows Tests +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS + +.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows + :alt: Windows .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy