From a35713e1dcb8a8223318b75e6f132332f65d31e8 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Mon, 31 Oct 2022 21:23:59 +0400 Subject: [PATCH 1/7] Bump Python To 3.11 Upgrade build image, configuration files, and documentation to python 3.11. Signed-off-by: Hassan Abouelela --- .github/CONTRIBUTING.md | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yaml | 2 +- .pre-commit-config.yaml | 2 +- Dockerfile | 4 ++-- config/snekbox.cfg | 6 +++--- pyproject.toml | 6 +++--- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b15a4ba7..6b70ce80 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -6,7 +6,7 @@ The Contributing Guidelines for Python Discord projects can be found [on our web ## Initial Setup -A Python 3.10 interpreter and `make` are required. A virtual environment is also recommended. Once that is set up, install the project's dependencies with `make setup`. +A Python 3.11 interpreter and `make` are required. A virtual environment is also recommended. Once that is set up, install the project's dependencies with `make setup`. This also installs a git pre-commit hook so that the linter runs upon a commit. Manual invocation is still possible with `make lint`. diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d53738b0..b4b69acb 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -17,7 +17,7 @@ jobs: id: python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" cache: pip cache-dependency-path: requirements/lint.pip diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 51eb0f81..189dc4c4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -65,7 +65,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.11" cache: pip cache-dependency-path: requirements/coveralls.pip diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 620c5bea..203e67ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: rev: 22.3.0 hooks: - id: black - language_version: "3.10" + language_version: "3.11" - repo: https://github.com/PyCQA/flake8 rev: &flake8_version 4.0.1 hooks: diff --git a/Dockerfile b/Dockerfile index 9816d54c..6355dac0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM python:3.10-slim-buster as builder +FROM python:3.11-slim-buster as builder WORKDIR /nsjail @@ -20,7 +20,7 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ RUN make # ------------------------------------------------------------------------------ -FROM python:3.10-slim-buster as base +FROM python:3.11-slim-buster as base # Everything will be a user install to allow snekbox's dependencies to be kept # separate from the packages exposed during eval. diff --git a/config/snekbox.cfg b/config/snekbox.cfg index a1caf680..87c216e6 100644 --- a/config/snekbox.cfg +++ b/config/snekbox.cfg @@ -14,7 +14,7 @@ envar: "OPENBLAS_NUM_THREADS=5" envar: "MKL_NUM_THREADS=5" envar: "VECLIB_MAXIMUM_THREADS=5" envar: "NUMEXPR_NUM_THREADS=5" -envar: "PYTHONPATH=/snekbox/user_base/lib/python3.10/site-packages" +envar: "PYTHONPATH=/snekbox/user_base/lib/python3.11/site-packages" envar: "PYTHONIOENCODING=utf-8:strict" keep_caps: false @@ -98,8 +98,8 @@ mount { } mount { - src: "/usr/local/bin/python3.10" - dst: "/usr/local/bin/python3.10" + src: "/usr/local/bin/python3.11" + dst: "/usr/local/bin/python3.11" is_bind: true rw: false } diff --git a/pyproject.toml b/pyproject.toml index e0a3d263..a1d1ea14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,14 +16,14 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", "Topic :: Security", "Topic :: Software Development :: Interpreters", ] dynamic = ["version"] -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ # Sentry's Falcon integration relies on api_helpers (falconry/falcon#1902). "falcon>=3.0.1", @@ -64,7 +64,7 @@ relative_files = true [tool.black] line-length = 100 -target-version = ["py310"] +target-version = ["py311"] force-exclude = "snekbox/config_pb2.py" [tool.isort] From 7a876b541c26ef5bc9c71144b25344ab76ede7e0 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Mon, 31 Oct 2022 21:27:03 +0400 Subject: [PATCH 2/7] Upgrade Package Dependencies Run `make upgrade` to relock all dependency files with pip-tools on 3.11. Signed-off-by: Hassan Abouelela --- requirements/coverage.pip | 6 ++---- requirements/coveralls.pip | 20 ++++++++------------ requirements/lint.pip | 19 ++++++++++--------- requirements/pip-tools.pip | 22 ++++++++++++++-------- requirements/requirements.pip | 16 ++++++++-------- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/requirements/coverage.pip b/requirements/coverage.pip index fcc14da7..e28a87ed 100644 --- a/requirements/coverage.pip +++ b/requirements/coverage.pip @@ -1,10 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.11 # To update, run: # # pip-compile --output-file=requirements/coverage.pip requirements/coverage.in # -coverage[toml]==6.4 +coverage[toml]==6.5.0 # via -r requirements/coverage.in -tomli==2.0.1 - # via coverage diff --git a/requirements/coveralls.pip b/requirements/coveralls.pip index cd16e7cd..20a59967 100644 --- a/requirements/coveralls.pip +++ b/requirements/coveralls.pip @@ -1,28 +1,24 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.11 # To update, run: # # pip-compile --output-file=requirements/coveralls.pip requirements/coveralls.in # -certifi==2022.5.18.1 +certifi==2022.9.24 # via requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via requests -coverage[toml]==6.4 +coverage[toml]==6.5.0 # via - # -c requirements/coverage.pip + # -c requirements\coverage.pip # coveralls coveralls==3.3.1 # via -r requirements/coveralls.in docopt==0.6.2 # via coveralls -idna==3.3 +idna==3.4 # via requests -requests==2.27.1 +requests==2.28.1 # via coveralls -tomli==2.0.1 - # via - # -c requirements/coverage.pip - # coverage -urllib3==1.26.9 +urllib3==1.26.12 # via requests diff --git a/requirements/lint.pip b/requirements/lint.pip index a399b6f0..e477369c 100644 --- a/requirements/lint.pip +++ b/requirements/lint.pip @@ -1,28 +1,29 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.11 # To update, run: # # pip-compile --output-file=requirements/lint.pip requirements/lint.in # cfgv==3.3.1 # via pre-commit -distlib==0.3.4 +distlib==0.3.6 # via virtualenv -filelock==3.7.0 +filelock==3.8.0 # via virtualenv -identify==2.5.1 +identify==2.5.8 # via pre-commit -nodeenv==1.6.0 +nodeenv==1.7.0 # via pre-commit platformdirs==2.5.2 # via virtualenv -pre-commit==2.19.0 +pre-commit==2.20.0 # via -r requirements/lint.in pyyaml==6.0 # via pre-commit -six==1.16.0 - # via virtualenv toml==0.10.2 # via pre-commit -virtualenv==20.14.1 +virtualenv==20.16.6 # via pre-commit + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/pip-tools.pip b/requirements/pip-tools.pip index 46d53d09..55bc0df6 100644 --- a/requirements/pip-tools.pip +++ b/requirements/pip-tools.pip @@ -1,19 +1,25 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.11 # To update, run: # # pip-compile --output-file=requirements/pip-tools.pip requirements/pip-tools.in # -click==8.1.3 +build==0.9.0 # via pip-tools -pep517==0.12.0 +click==8.1.3 # via pip-tools -pip-tools==6.6.2 - # via -r requirements/pip-tools.in -tomli==2.0.1 +colorama==0.4.6 # via - # -c requirements/coverage.pip - # pep517 + # build + # click +packaging==21.3 + # via build +pep517==0.13.0 + # via build +pip-tools==6.9.0 + # via -r requirements/pip-tools.in +pyparsing==3.0.9 + # via packaging wheel==0.37.1 # via pip-tools diff --git a/requirements/requirements.pip b/requirements/requirements.pip index 034f1048..afa77469 100644 --- a/requirements/requirements.pip +++ b/requirements/requirements.pip @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.11 # To update, run: # # pip-compile --extra=gunicorn --extra=sentry --output-file=requirements/requirements.pip pyproject.toml # -attrs==21.4.0 +attrs==22.1.0 # via jsonschema -certifi==2022.5.18.1 +certifi==2022.9.24 # via sentry-sdk falcon==3.1.0 # via @@ -14,15 +14,15 @@ falcon==3.1.0 # snekbox (pyproject.toml) gunicorn==20.1.0 # via snekbox (pyproject.toml) -jsonschema==4.5.1 +jsonschema==4.16.0 # via snekbox (pyproject.toml) -protobuf==4.21.1 +protobuf==4.21.9 # via snekbox (pyproject.toml) -pyrsistent==0.18.1 +pyrsistent==0.19.1 # via jsonschema -sentry-sdk[falcon]==1.5.12 +sentry-sdk[falcon]==1.10.1 # via snekbox (pyproject.toml) -urllib3==1.26.9 +urllib3==1.26.12 # via sentry-sdk # The following packages are considered to be unsafe in a requirements file: From 531d0e2bf5fd95ae955efb8589e234d9db306901 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Mon, 31 Oct 2022 21:50:55 +0400 Subject: [PATCH 3/7] Drop Python Coveralls Dependency Remove the coveralls dependency in favor of the GitHub action uploader. The action is an official product from coveralls, and has the same features as the package, however it reduces the project's dependencies in favor of a simpler more stand-alone solution. Signed-off-by: Hassan Abouelela --- .github/workflows/test.yaml | 14 +++++++++----- Makefile | 1 - requirements/coveralls.in | 3 --- requirements/coveralls.pip | 24 ------------------------ 4 files changed, 9 insertions(+), 33 deletions(-) delete mode 100644 requirements/coveralls.in delete mode 100644 requirements/coveralls.pip diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 189dc4c4..9e30d491 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -67,10 +67,10 @@ jobs: with: python-version: "3.11" cache: pip - cache-dependency-path: requirements/coveralls.pip + cache-dependency-path: requirements/coverage.pip - name: Install dependencies - run: pip install -U -r requirements/coveralls.pip + run: pip install -U -r requirements/coverage.pip - name: Download coverage data uses: actions/download-artifact@v2 @@ -83,9 +83,13 @@ jobs: - name: Display coverage report run: coverage report -m + - name: Generate lcov report + run: coverage lcov + # Comment on the PR with the coverage results and register a GitHub check # which links to the coveralls.io job. - name: Publish coverage report to coveralls.io - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls --service=github + uses: coverallsapp/github-action@1.1.3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: ./coverage.lcov diff --git a/Makefile b/Makefile index a30579a0..7242c25b 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,6 @@ upgrade: install-piptools $(PIP_COMPILE_CMD) -o requirements/requirements.pip \ --extra gunicorn --extra sentry pyproject.toml $(PIP_COMPILE_CMD) -o requirements/coverage.pip requirements/coverage.in - $(PIP_COMPILE_CMD) -o requirements/coveralls.pip requirements/coveralls.in $(PIP_COMPILE_CMD) -o requirements/lint.pip requirements/lint.in $(PIP_COMPILE_CMD) -o requirements/pip-tools.pip requirements/pip-tools.in diff --git a/requirements/coveralls.in b/requirements/coveralls.in deleted file mode 100644 index 28d49590..00000000 --- a/requirements/coveralls.in +++ /dev/null @@ -1,3 +0,0 @@ --c coverage.pip - -coveralls>=3.3.1 diff --git a/requirements/coveralls.pip b/requirements/coveralls.pip deleted file mode 100644 index 20a59967..00000000 --- a/requirements/coveralls.pip +++ /dev/null @@ -1,24 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.11 -# To update, run: -# -# pip-compile --output-file=requirements/coveralls.pip requirements/coveralls.in -# -certifi==2022.9.24 - # via requests -charset-normalizer==2.1.1 - # via requests -coverage[toml]==6.5.0 - # via - # -c requirements\coverage.pip - # coveralls -coveralls==3.3.1 - # via -r requirements/coveralls.in -docopt==0.6.2 - # via coveralls -idna==3.4 - # via requests -requests==2.28.1 - # via coveralls -urllib3==1.26.12 - # via requests From f18b4c5730c65d3326efd3ec55a0b6e30f2524e6 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Mon, 31 Oct 2022 22:22:10 +0400 Subject: [PATCH 4/7] Bump Pre-Commit Hooks Signed-off-by: Hassan Abouelela --- .pre-commit-config.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 203e67ab..26dad6a4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.3.0 hooks: - id: check-merge-conflict - id: check-toml @@ -17,18 +17,17 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 22.10.0 hooks: - id: black - language_version: "3.11" - repo: https://github.com/PyCQA/flake8 - rev: &flake8_version 4.0.1 + rev: &flake8_version 5.0.4 hooks: - &flake8_hook id: flake8 additional_dependencies: - flake8-annotations~=2.7 - - flake8-bugbear==22.4.25 + - flake8-bugbear==22.10.27 - flake8-docstrings~=1.4 - flake8-string-format~=0.3.0 - flake8-todo~=0.7 From 9aa2c3a840b7e83aca1c5e90e65eb4e749191607 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Mon, 31 Oct 2022 23:05:54 +0400 Subject: [PATCH 5/7] Upgrade Runtime Packages Signed-off-by: Hassan Abouelela --- deployment.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deployment.yaml b/deployment.yaml index ffba3866..496b9e58 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -40,17 +40,17 @@ spec: forbiddenfruit~=0.1 fuzzywuzzy~=0.18 lark~=1.1 - more-itertools~=8.14 + more-itertools~=9.0 networkx~=2.8 numpy~=1.23 - pandas~=1.4 + pandas~=1.5 pendulum~=2.1 python-dateutil~=2.8 pyyaml~=6.0 - sympy~=1.10 + sympy~=1.11 toml~=0.10 - typing-extensions~=4.3 - tzdata~=2022.2 + typing-extensions~=4.4 + tzdata~=2022.6 yarl~=1.8 volumes: - name: snekbox-user-base-volume From ce92e4d1eed47b290f3b9d4f3a4d6ed565ef3efb Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Mon, 31 Oct 2022 23:19:04 +0400 Subject: [PATCH 6/7] Switch To Compose v2 The compose file already uses compose v2 features, and most clients likely already alias docker-compose to v2 under the hood. This change is more of a formality. Signed-off-by: Hassan Abouelela --- .github/CONTRIBUTING.md | 4 +++- .github/workflows/test.yaml | 2 +- Makefile | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 6b70ce80..d0a69216 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -13,7 +13,8 @@ Manual invocation is still possible with `make lint`. ## Running snekbox -Use `docker-compose up` to start snekbox in development mode. The optional `--build` argument can be passed to force the image to be rebuilt. +Use `docker compose up` to start snekbox in development mode. The optional `--build` argument can be passed to force the image to be rebuilt. +You must use [compose v2][Compose v2], accessed via `docker compose` (no hyphen). The container has all development dependencies. The repository on the host is mounted within the container; changes made to local files will also affect the container. @@ -63,3 +64,4 @@ Other things to look out for are breaking changes to NsJail's config format, its [readme]: ../README.md [Dockerfile]: ../Dockerfile +[Compose v2]: https://docs.docker.com/compose/compose-v2/ diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9e30d491..97fb566c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,7 +39,7 @@ jobs: id: run_tests run: | export IMAGE_SUFFIX='-venv:${{ inputs.version }}' - docker-compose run \ + docker compose run \ --rm -T -e COVERAGE_DATAFILE=.coverage.${{ matrix.os }} \ --entrypoint coverage \ snekbox \ diff --git a/Makefile b/Makefile index 7242c25b..a385a8eb 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,8 @@ lint: setup # Fix ownership of the coverage file even if tests fail & preserve exit code .PHONY: test test: - docker-compose build -q --force-rm - docker-compose run --entrypoint /bin/bash --rm snekbox -c \ + docker compose build -q --force-rm + docker compose run --entrypoint /bin/bash --rm snekbox -c \ 'coverage run -m unittest; e=$?; chown --reference=. .coverage; exit $e' .PHONY: report @@ -40,4 +40,4 @@ build: .PHONY: devsh devsh: - docker-compose run --entrypoint /bin/bash --rm snekbox + docker compose run --entrypoint /bin/bash --rm snekbox From 09191679abfeb743da3a2f93ff642d7e7ff1f46a Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Mon, 31 Oct 2022 23:40:52 +0400 Subject: [PATCH 7/7] Fix Invalid GitHub Action One of the GitHub action files was failing validation due to an unindented block. It is unclear why this failure started now. Signed-off-by: Hassan Abouelela --- .github/workflows/lint.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index b4b69acb..cccdb898 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -29,8 +29,8 @@ jobs: with: path: ${{ env.PRE_COMMIT_HOME }} key: "precommit-0-${{ runner.os }}-${{ env.PRE_COMMIT_HOME }}-\ - ${{ steps.python.outputs.python-version }}-\ - ${{ hashFiles('./.pre-commit-config.yaml') }}" + ${{ steps.python.outputs.python-version }}-\ + ${{ hashFiles('./.pre-commit-config.yaml') }}" # Skip the flake8 hook because the following command will run it. - name: Run pre-commit hooks