From 2d24b5650f64f371e60885ab8dc4dd22ec9d6ecc Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Wed, 7 Dec 2022 21:36:37 -0500 Subject: [PATCH 1/8] Update dev requirements This updates dev requirements. While doing that, I split out flake8 into its own world because its dependencies conflict with other dependencies. --- MANIFEST.in | 4 +++- requirements-dev.txt | 19 +++++++++---------- requirements-flake8.txt | 2 ++ scripts/run_tests.sh | 7 ------- tox.ini | 2 +- 5 files changed, 15 insertions(+), 19 deletions(-) create mode 100644 requirements-flake8.txt diff --git a/MANIFEST.in b/MANIFEST.in index aa1088d6..39d79b6a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,6 +6,8 @@ include SECURITY.md include tox.ini include LICENSE include README.rst +include requirements-dev.txt +include requirements-flake8.txt include docs/conf.py include docs/Makefile @@ -13,7 +15,7 @@ include docs/requirements.txt include scripts/* -recursive-include bleach *.py *.json *.rst *.sh *.txt INSTALLER METADATA RECORD WHEEL LICENSE REQUESTED *.SHA256SUM +recursive-include bleach *.py *.rst *.sh *.txt INSTALLER METADATA RECORD WHEEL LICENSE REQUESTED *.SHA256SUM recursive-include docs *.rst recursive-include tests *.py *.test recursive-include tests_website *.html *.py *.rst diff --git a/requirements-dev.txt b/requirements-dev.txt index 27f9aed8..bc092ed7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,11 @@ -e . -black==22.3.0; implementation_name == 'cpython' -build==0.8.0 -flake8==4.0.1 -mypy==0.961; implementation_name =='cpython' -pip-tools==6.6.2 -pytest==7.1.2 -Sphinx==4.3.2 -tox==3.25.0 -twine==4.0.1 -wheel==0.37.1 +black==22.10.0; implementation_name == 'cpython' +build==0.9.0 +mypy==0.991; implementation_name =='cpython' +pip-tools==6.11.0 +pytest==7.2.0 +Sphinx==5.3.0 +tox==4.0.2 +twine==4.0.2 +wheel==0.38.4 diff --git a/requirements-flake8.txt b/requirements-flake8.txt new file mode 100644 index 00000000..806cdff7 --- /dev/null +++ b/requirements-flake8.txt @@ -0,0 +1,2 @@ +# Requirements for running flake8 +flake8==6.0.0 diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 0705d5e1..0d3fc917 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -28,13 +28,6 @@ case "${MODE}" in format-check) black --target-version=py37 --check --diff --exclude=_vendor setup.py bleach/ tests/ tests_website/ ;; - check-reqs) - python -m venv ./tmpvenv/ - ./tmpvenv/bin/pip install -U pip - ./tmpvenv/bin/pip install '.[dev,css]' - ./tmpvenv/bin/pip list -o - rm -rf ./tmpvenv/ - ;; *) echo "Unknown mode $MODE." exit 1 diff --git a/tox.ini b/tox.ini index e3a22ebb..c6cfa6bf 100644 --- a/tox.ini +++ b/tox.ini @@ -85,7 +85,7 @@ commands = [testenv:lint] basepython = python3.9 changedir = scripts -deps = -rrequirements-dev.txt +deps = -rrequirements-flake8.txt commands = ./run_tests.sh lint From bf2f2ce6d443cb7c4334eec2df5cf434c5c1d22f Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Wed, 7 Dec 2022 21:51:22 -0500 Subject: [PATCH 2/8] Fix lint workflow This fixes the lint workflow to run the tox environments which build the correct environment and then runs the script inside it. --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 658cb2e7..c5656361 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,4 +37,4 @@ jobs: - name: Tests shell: bash - run: ./scripts/run_tests.sh ${{ matrix.mode }} + run: tox -e ${{ matrix.mode }} From 4d6eae5a97ed7d1420eedf0a4bb3671afbf28fb1 Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Wed, 7 Dec 2022 23:31:03 -0500 Subject: [PATCH 3/8] Redo linting and testing in CI This redoes how linting and testing work in CI. This uses tox-gh-actions to run tox in the right python environments. It switches the linting items to the test workflow because they're defined as tox environments. --- .github/workflows/lint.yml | 40 -------------------------------------- .github/workflows/test.yml | 2 +- requirements-dev.txt | 3 ++- tox.ini | 33 +++++++++++++++++++------------ 4 files changed, 24 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index c5656361..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -name: Lint - -on: [push, pull_request] - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: [3.9] - os: [ubuntu-latest] - mode: [lint, vendorverify, docs, format-check] - - steps: - - uses: actions/checkout@v3 - - - name: Cache - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: - ${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ matrix.os }}-${{ matrix.python-version }}- - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install -U pip setuptools>=18.5 - python -m pip install -r requirements-dev.txt - - - name: Tests - shell: bash - run: tox -e ${{ matrix.mode }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c6489fa5..8de398e3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,4 +42,4 @@ jobs: - name: Tests shell: bash - run: ./scripts/run_tests.sh + run: tox diff --git a/requirements-dev.txt b/requirements-dev.txt index bc092ed7..400cf874 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,7 @@ mypy==0.991; implementation_name =='cpython' pip-tools==6.11.0 pytest==7.2.0 Sphinx==5.3.0 -tox==4.0.2 +tox==3.27.1 +tox-gh-actions==2.11.0 twine==4.0.2 wheel==0.38.4 diff --git a/tox.ini b/tox.ini index c6cfa6bf..0c256d56 100644 --- a/tox.ini +++ b/tox.ini @@ -5,10 +5,19 @@ envlist = py{37,38,39,310,311,py3} py{37,38,39,310,311,py3}-tinycss2 py{37,38,39,310,311}-build-no-lang - docs - format-check - lint - vendorverify + py39-docs + py39-format-check + py39-lint + py39-vendorverify + +[gh-actions] +python = + 3.7: py37 + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + py3: pypy-3.8 [testenv] deps = -rrequirements-dev.txt @@ -82,32 +91,32 @@ setenv = commands = python setup.py build -[testenv:lint] -basepython = python3.9 +[testenv:py39-lint] changedir = scripts deps = -rrequirements-flake8.txt +platform = linux commands = ./run_tests.sh lint -[testenv:vendorverify] -basepython = python3.9 +[testenv:py39-vendorverify] changedir = scripts deps = -rrequirements-dev.txt +platform = linux commands = ./run_tests.sh vendorverify -[testenv:format-check] -basepython = python3.9 +[testenv:py39-format-check] changedir = scripts deps = -rrequirements-dev.txt +platform = linux commands = ./run_tests.sh format-check -[testenv:docs] -basepython = python3.9 +[testenv:py39-docs] changedir = docs deps = -rrequirements-dev.txt extras = css +platform = linux commands = sphinx-build -b html -d {envtmpdir}/doctrees . {envtmpdir}/html sphinx-build -b doctest -d {envtmpdir}/doctrees . {envtmpdir}/doctest From 2e5e5757b0ef4a26d4e2539739273fd9db731d88 Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Thu, 8 Dec 2022 16:01:48 -0500 Subject: [PATCH 4/8] Rework testing and CI to be tox-based This fixes the issue where a bunch of tox-defined environments weren't being run in CI, so there was a whole bunch of testing that wasn't happening on pull requests. Now everything is tox-based and CI runs all the tox environments by Python environment using tox-gh-actions. --- .github/workflows/test.yml | 2 +- Makefile | 33 +++++++++++++++++++++++++++++++++ docs/dev.rst | 19 +++++++++---------- tox.ini | 12 ++++++------ 4 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 Makefile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8de398e3..32b539ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,6 @@ jobs: python -m pip install -U pip setuptools>=18.5 python -m pip install -r requirements-dev.txt - - name: Tests + - name: Run tests, linting, and everything else shell: bash run: tox diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..a36082a7 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +DEFAULT_GOAL := help +PROJECT=bleach + +.PHONY: help +help: + @echo "Available rules:" + @fgrep -h "##" Makefile | fgrep -v fgrep | sed 's/\(.*\):.*##/\1: /' + +.PHONY: test +test: ## Run tests + tox + +.PHONY: docs +docs: ## Build docs + tox -e py39-docs + +.PHONY: lint +lint: ## Lint files + black --target-version=py37 --exclude=_vendor setup.py bleach/ tests/ tests_website/ + tox -e py39-lint + tox -e py39-format-check + +.PHONY: vendorverify +vendorverify: ## Verify vendored files + tox -e py39-vendorverify + +.PHONY: clean +clean: ## Clean build artifacts + rm -rf build dist ${PROJECT}.egg-info .tox .pytest_cache + rm -rf docs/_build/* + rm -rf .eggs + find . -name __pycache__ | xargs rm -rf + find . -name '*.pyc' | xargs rm -rf diff --git a/docs/dev.rst b/docs/dev.rst index f0d67e86..ff33d27a 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -42,10 +42,10 @@ Run:: $ tox -That'll run Bleach tests in all the supported Python environments. Note -that you need the necessary Python binaries for them all to be tested. +That'll run Bleach tests in all the supported Python environments. Note that +you need the necessary Python binaries for them all to be tested. -Tests are run as github actions for test and pull request events. +Tests are run as GitHub actions for test and pull request events. Release process @@ -67,7 +67,7 @@ Release process 5. Verify correctness. - 1. Run tests with tox:: + 1. Run linting, tests, and everything else with tox:: $ tox @@ -85,8 +85,7 @@ Release process exit with ``/tmp/vendor-test exists. Please remove.`` and the exit code should be zero):: - $ ./scripts/run_tests.sh vendorverify - $ ./scripts/run_tests.sh vendorverify + $ make vendorverify 5. Run any additional tests to verify everything else works @@ -102,7 +101,7 @@ Release process 9. Generate distribution files:: - $ python setup.py sdist bdist_wheel + $ python -m build 10. Sanity check the release contents and sizes:: @@ -110,13 +109,13 @@ Release process $ tar tvzf dist/bleach-${VERSION}.tar.gz $ unzip -v dist/bleach-${VERSION}-py2.py3-none-any.whl -11. Upload them to PyPI:: +11. Using a PyPI API token, upload dist files to PyPI:: - $ twine upload dist/* + $ twine upload -r [REPO] dist/* 12. Push the new tag:: - $ git push --tags official main + $ git push --tags [REMOTE] main That will push the release to PyPI. diff --git a/tox.ini b/tox.ini index 0c256d56..b5b6b764 100644 --- a/tox.ini +++ b/tox.ini @@ -92,25 +92,25 @@ commands = python setup.py build [testenv:py39-lint] -changedir = scripts +changedir = {toxinidir} deps = -rrequirements-flake8.txt platform = linux commands = - ./run_tests.sh lint + flake8 setup.py tests/ bleach/ tests_website/ [testenv:py39-vendorverify] -changedir = scripts +changedir = {toxinidir} deps = -rrequirements-dev.txt platform = linux commands = - ./run_tests.sh vendorverify + {toxinidir}/scripts/vendor_verify.sh [testenv:py39-format-check] -changedir = scripts +changedir = {toxinidir} deps = -rrequirements-dev.txt platform = linux commands = - ./run_tests.sh format-check + black --target-version=py37 --check --diff --exclude=_vendor setup.py bleach/ tests/ tests_website/ [testenv:py39-docs] changedir = docs From 79fef51b9e1a6580120c49e28be2a16e99309825 Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Thu, 8 Dec 2022 16:20:35 -0500 Subject: [PATCH 5/8] Add Makefile to MANIFEST.in --- MANIFEST.in | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 39d79b6a..a2784f86 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,13 +1,14 @@ include CHANGES -include CONTRIBUTORS -include CONTRIBUTING.rst include CODE_OF_CONDUCT.md -include SECURITY.md -include tox.ini +include CONTRIBUTING.rst +include CONTRIBUTORS include LICENSE +include Makefile include README.rst include requirements-dev.txt include requirements-flake8.txt +include SECURITY.md +include tox.ini include docs/conf.py include docs/Makefile From 1c322eca0716a68bcee8dca23aba7a86c701f467 Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Thu, 8 Dec 2022 16:22:28 -0500 Subject: [PATCH 6/8] Remove scripts/run_tests.sh since we don't use it anymore --- scripts/run_tests.sh | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100755 scripts/run_tests.sh diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh deleted file mode 100755 index 0d3fc917..00000000 --- a/scripts/run_tests.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Make sure we're running from the bleach repository directory and -# not this directory. -THISDIR=$(basename `pwd`) -if [[ "${THISDIR}" == "scripts" ]]; then - cd .. -fi - -MODE=${1:-test} - -case "${MODE}" in - test) - pytest - ;; - lint) - flake8 setup.py tests/ bleach/ tests_website/ - ;; - vendorverify) - ./scripts/vendor_verify.sh - ;; - docs) - tox -e docs - ;; - format) - black --target-version=py37 --exclude=_vendor setup.py bleach/ tests/ tests_website/ - ;; - format-check) - black --target-version=py37 --check --diff --exclude=_vendor setup.py bleach/ tests/ tests_website/ - ;; - *) - echo "Unknown mode $MODE." - exit 1 -esac From cc1ce87ca77a018d58a8e51c59e349ae271f7f8b Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Thu, 8 Dec 2022 16:38:49 -0500 Subject: [PATCH 7/8] Fix CI for pypy --- .github/workflows/test.yml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 32b539ee..94e52ed2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.8"] os: [ubuntu-latest, macos-latest, windows-latest] steps: diff --git a/tox.ini b/tox.ini index b5b6b764..7cb5f282 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ python = 3.9: py39 3.10: py310 3.11: py311 - py3: pypy-3.8 + pypy-3.8: pypy3 [testenv] deps = -rrequirements-dev.txt @@ -55,7 +55,7 @@ extras = css commands = pytest {posargs:-v} -[testenv:py3-tinycss2] +[testenv:pypy3-tinycss2] deps = -rrequirements-dev.txt extras = css commands = From 92ecbf2b3938454329371933880c405a9beda3ca Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Thu, 8 Dec 2022 16:55:55 -0500 Subject: [PATCH 8/8] Update cache action to v3 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94e52ed2..d4a6bd09 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,7 @@ jobs: echo "::set-output name=dir::$(pip cache dir)" - name: Cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: