From 2538f5f51e6d5bedd7be1bdee349417da38ded35 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 18 Dec 2025 01:13:30 +0100 Subject: [PATCH] Switch CI to UV GitHub actions --- .github/FUNDING.yml | 2 - .github/dependabot.yml | 24 +++---- .github/workflows/ci.yml | 127 +++++----------------------------- .github/workflows/release.yml | 40 ++++++----- .gitignore | 3 + .pre-commit-config.yaml | 45 ++++++++++++ LICENSE | 1 - MANIFEST.in | 4 -- README.md | 29 ++++---- SECURITY.md | 54 --------------- linter-requirements.txt | 5 -- pyproject.toml | 59 +++++++++++++--- s3file/forms.py | 4 +- s3file/storages_optimized.py | 4 +- s3file/views.py | 2 +- setup.cfg | 4 -- tests/conftest.py | 2 +- tests/test_forms.py | 14 ++-- tests/testapp/urls.py | 20 +++--- 19 files changed, 183 insertions(+), 260 deletions(-) delete mode 100644 .github/FUNDING.yml create mode 100644 .pre-commit-config.yaml delete mode 100644 MANIFEST.in delete mode 100644 SECURITY.md delete mode 100644 linter-requirements.txt delete mode 100644 setup.cfg diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index e030a5a..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,2 +0,0 @@ -github: codingjoe -custom: https://www.paypal.me/codingjoe diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 01e1893..52c1116 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,14 @@ version: 2 updates: -- package-ecosystem: pip - directory: "/" - schedule: - interval: daily -- package-ecosystem: npm - directory: "/" - schedule: - interval: weekly -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: daily + - package-ecosystem: pip + directory: "/" + schedule: + interval: daily + - package-ecosystem: npm + directory: "/" + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbf3812..356ecef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,24 +1,20 @@ name: CI - on: push: branches: - main pull_request: - jobs: - dist: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: astral-sh/setup-uv@v7 + - run: uvx --from build pyproject-build --sdist --wheel + - run: uvx twine check dist/* + - uses: actions/upload-artifact@v6 with: - python-version: "3.x" - - run: python -m pip install --upgrade pip build wheel twine - - run: python -m build --sdist --wheel - - run: python -m twine check dist/* - + path: dist/* js-lint: runs-on: ubuntu-latest steps: @@ -29,8 +25,6 @@ jobs: - name: Install Node dependencies run: npm ci - run: npm run lint:js - - js-test: runs-on: ubuntu-latest needs: @@ -48,30 +42,8 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} flags: javascript file: lcov.txt - - py-lint: - runs-on: ubuntu-latest - strategy: - matrix: - lint-command: - - bandit -r . -x ./tests - - black --check --diff . - - flake8 . - - isort --check-only --diff . - - pydocstyle . - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - cache: 'pip' - cache-dependency-path: 'linter-requirements.txt' - - run: python -m pip install -r linter-requirements.txt - - run: ${{ matrix.lint-command }} - pytest: needs: - - py-lint - dist runs-on: ubuntu-latest strategy: @@ -85,91 +57,28 @@ jobs: - "5.0" - "5.1" steps: - - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Chrome - run: | - sudo apt update - sudo apt install -y google-chrome-stable - - name: Install Selenium - run: | - mkdir bin - curl -qO "https://chromedriver.storage.googleapis.com/$(curl -q https://chromedriver.storage.googleapis.com/LATEST_RELEASE)/chromedriver_linux64.zip" - unzip chromedriver_linux64.zip -d bin - - - run: python -m pip install .[test] - - run: python -m pip install django~=${{ matrix.django-version }}.0 - - run: python -m pytest -m "not selenium" - env: - PATH: $PATH:$(pwd)/bin - - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - flags: python - - + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.python-version }} + - run: uv run --with django~=${{ matrix.django-version }}.0 pytest -m "not selenium" + - uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: python selenium: needs: - pytest - strategy: - matrix: - python-version: - - "3.x" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - name: Install Chrome - run: sudo apt-get install -y google-chrome-stable - name: Install Selenium run: | - mkdir bin - curl -O https://chromedriver.storage.googleapis.com/`curl -s https://chromedriver.storage.googleapis.com/LATEST_RELEASE`/chromedriver_linux64.zip - unzip chromedriver_linux64.zip -d bin - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - run: python -m pip install -e .[test] - - run: python -m pytest -m selenium + wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + sudo dpkg -i google-chrome-stable_current_amd64.deb || sudo apt-get -f install -y + - uses: astral-sh/setup-uv@v7 + - run: uv run pytest -m selenium - uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} flags: selenium - - - analyze: - name: CodeQL Analyze - needs: - - pytest - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ javascript, python ] - - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: ${{ matrix.language }} - queries: +security-and-quality - - - name: Autobuild - uses: github/codeql-action/autobuild@v4 - if: ${{ matrix.language == 'javascript' || matrix.language == 'python' }} - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 080a84c..f6eabda 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,26 +1,28 @@ name: Release - on: release: types: [published] - + workflow_dispatch: jobs: - PyPi: - + pypi-build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - run: python -m pip install --upgrade pip build wheel twine - - uses: actions/setup-node@v6 - - name: Install Node dependencies - run: npm ci - - name: Minify JavaScript files - run: npm run minify - - run: python -m build --sdist --wheel - - run: python -m twine upload dist/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v7 + - run: uvx --from build pyproject-build --sdist --wheel + - uses: actions/upload-artifact@v6 + with: + name: release-dists + path: dist/ + pypi-publish: + runs-on: ubuntu-latest + needs: + - pypi-build + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v7 + with: + name: release-dists + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index cf768ef..f49b007 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ target/ node_modules *.min.js + +# uv +uv.lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..01f6a5e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: check-merge-conflict + - id: check-ast + - id: check-toml + - id: check-yaml + - id: check-symlinks + - id: debug-statements + - id: end-of-file-fixer + - id: no-commit-to-branch + args: [--branch, main] + - repo: https://github.com/asottile/pyupgrade + rev: v3.21.2 + hooks: + - id: pyupgrade + - repo: https://github.com/adamchainz/django-upgrade + rev: 1.29.1 + hooks: + - id: django-upgrade + - repo: https://github.com/hukkin/mdformat + rev: 1.0.0 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-ruff + - mdformat-footnote + - mdformat-gfm + - mdformat-gfm-alerts + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.9 + hooks: + - id: ruff-check + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + - repo: https://github.com/google/yamlfmt + rev: v0.20.0 + hooks: + - id: yamlfmt +ci: + autoupdate_schedule: weekly + skip: + - no-commit-to-branch diff --git a/LICENSE b/LICENSE index 49b017b..25867f7 100644 --- a/LICENSE +++ b/LICENSE @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index ea0bdae..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -include s3file/static/s3file/js/s3file.js s3file/static/s3file/js/s3file.min.js -prune tests -prune .github -exclude .* diff --git a/README.md b/README.md index 90eda41..838a73f 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,11 @@ license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubu ## Features -- lightweight: less 200 lines -- no JavaScript or Python dependencies (no jQuery) -- easy integration -- works just like the built-in -- extendable JavaScript API +- lightweight: less 200 lines +- no JavaScript or Python dependencies (no jQuery) +- easy integration +- works just like the built-in +- extendable JavaScript API ## For the Nerds @@ -81,15 +81,15 @@ Add the S3File app and middleware in your settings: # settings.py INSTALLED_APPS = ( - '...', - 's3file', - '...', + "...", + "s3file", + "...", ) MIDDLEWARE = ( - '...', - 's3file.middleware.S3FileMiddleware', - '...', + "...", + "s3file.middleware.S3FileMiddleware", + "...", ) ``` @@ -150,7 +150,7 @@ to `FileSystemStorage`. To prevent users from accidentally using the `FileSystemStorage` and the insecure S3 dummy backend in production, there is also an additional -deployment check that will error if you run Django\'s deployment check +deployment check that will error if you run Django's deployment check suite: ```shell @@ -177,7 +177,7 @@ uploading the file to S3 and then copying it byte-by-byte to perform a move operation just to rename the uploaded object. For large files this leads to additional loading times for the user. -That\'s why S3File provides an optimized version of this method at +That's why S3File provides an optimized version of this method at `storages_optimized.S3OptimizedUploadStorage`. It uses the more efficient `copy` method from S3, given that we know that we only copy from one S3 location to another. @@ -185,6 +185,7 @@ from one S3 location to another. ```python from s3file.storages_optimized import S3OptimizedUploadStorage + class MyStorage(S3OptimizedUploadStorage): # Subclass and use like any other storage - default_acl = 'private' + default_acl = "private" ``` diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index cf70d2e..0000000 --- a/SECURITY.md +++ /dev/null @@ -1,54 +0,0 @@ -# Security Policy - -## Security Considerations - -The wake of CVE-2022-24840 revealed the importance to document security considerations. -The following attack vectors have been considered during development. Should there be -a possible vector or consideration missing, please contact the maintainers, as described -below. - -We use [pre-signed POST URLs](s3-pre-signed-url) to upload files to AWS S3. -[Django's internal signer](django-signing) is used to sign the upload path and validate -it before fetching files from S3. - -Please note, that Django's signer uses the `SECRET_KEY`, rotating the key will void all -signatures. Should you rotate the secret key, between a form GET and POST request, the -form will fail. Similarly, Django will expire all sessions if you rotate the key. - -[s3-pre-signed-url]: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-presigned-urls.html -[django-signing]: https://docs.djangoproject.com/en/stable/topics/signing/ - -### Upload of malicious files - -AWS S3 supports MIME type detection and content-type enforcement. -You can limit the upload of malicious files via the MIME type [accept][accept]. -However, this is not a security measure, and you should always validate files before -processing them. - -[accept]: https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes/accept - -### Request file injection - -Though files can always be included in a request, CVE-2022-24840 revealed that we need -to consider people injecting any files that reside on your S3 bucket. However, we do -presign the upload location and validate it before fetching files from S3. - -### Path traversal & timing attacks - -We fetch files from your S3 bucket. This behavior could be used to brute force valid -file names. We mitigate this by signing the allowed upload path and validating it. -The upload path is unique for each file input and request. Therefore, an attacker can -not escape and access any files but the one uploaded by the attacker. - -## Reporting a Vulnerability - -NEVER open an issue or discussion to report a vulnerability. - -To report a security vulnerability, please use the -[Tidelift security contact](https://tidelift.com/security). -Tidelift will coordinate the fix and disclosure. - -You may also contact one of the maintainers of the project either via email or Telegram: - -* Email: [johannes@maron.family](mailto:johannes@maron.family) -* Telegram: [@codingjoe](https://t.me/codingjoe) diff --git a/linter-requirements.txt b/linter-requirements.txt deleted file mode 100644 index 68aafbf..0000000 --- a/linter-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -bandit==1.9.2 -black==25.12.0 -flake8==7.3.0 -isort==7.0.0 -pydocstyle[toml]==6.3.0 diff --git a/pyproject.toml b/pyproject.toml index edde257..c6db4d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,10 @@ dependencies = [ "boto3", ] -[project.optional-dependencies] +[dependency-groups] +dev = [ + { include-group = "test" }, +] test = [ "pytest >=2.7.3", "pytest-cov", @@ -61,13 +64,49 @@ testpaths = [ ] DJANGO_SETTINGS_MODULE = "tests.testapp.settings" -[tool.isort] -atomic = true -line_length = 88 -known_first_party = "s3file, tests" -include_trailing_comma = true -default_section = "THIRDPARTY" -combine_as_imports = true +[tool.ruff] +src = ["s3file", "tests"] +line-length = 88 +indent-width = 4 + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" +preview = true + +[tool.ruff.lint] +select = [ + "D", # pydocstyle + "E", # pycodestyle errors + "EXE", # flake8-executable + "F", # pyflakes + "I", # isort + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "RET", # flake8-return + "S", # flake8-bandit + "SIM", # flake8-simplify + "UP", # pyupgrade + "W", # pycodestyle warnings +] + +ignore = ["D1", "E501", "PT012"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["S"] + +[tool.ruff.lint.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false + +[tool.ruff.lint.isort] +combine-as-imports = true +split-on-trailing-comma = true +section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"] +force-wrap-aliases = true +known-first-party = ["s3file", "tests"] -[tool.pydocstyle] -add_ignore = "D1" +[tool.ruff.lint.pydocstyle] +convention = "pep257" diff --git a/s3file/forms.py b/s3file/forms.py index a5006bb..a9f3453 100644 --- a/s3file/forms.py +++ b/s3file/forms.py @@ -86,7 +86,7 @@ def build_attrs(self, *args, **kwargs): ) defaults = { - "data-fields-%s" % key: value for key, value in response["fields"].items() + f"data-fields-{key}": value for key, value in response["fields"].items() } defaults["data-url"] = response["url"] # we sign upload location, and will only accept files within the same folder @@ -106,7 +106,7 @@ def get_conditions(self, accept): if accept and "," not in accept: top_type, sub_type = accept.split("/", 1) if sub_type == "*": - conditions.append(["starts-with", "$Content-Type", "%s/" % top_type]) + conditions.append(["starts-with", "$Content-Type", f"{top_type}/"]) else: conditions.append({"Content-Type": accept}) else: diff --git a/s3file/storages_optimized.py b/s3file/storages_optimized.py index e1a0597..38984a9 100644 --- a/s3file/storages_optimized.py +++ b/s3file/storages_optimized.py @@ -6,10 +6,10 @@ class S3OptimizedUploadStorage(S3Boto3Storage): """ Class for an optimized S3 storage. - This storage prevents unnecessary operation to copy with the general ``upload_fileobj`` + This storage prevents unnecessary operation to copy with the general `upload_fileobj` command when the object already is a S3 object where the faster copy command can be used. - The assumption is that ``content`` contains a S3 object from which we can copy. + The assumption is that `content` contains a S3 object from which we can copy. See also discussion here: https://github.com/codingjoe/django-s3file/discussions/126 """ diff --git a/s3file/views.py b/s3file/views.py index f25d24b..10c4486 100644 --- a/s3file/views.py +++ b/s3file/views.py @@ -40,7 +40,7 @@ def post(self, request): return http.HttpResponseForbidden() key = key.replace("${filename}", file.name) - etag = hashlib.md5(file.read()).hexdigest() # nosec + etag = hashlib.md5(file.read()).hexdigest() # noqa: S324 file.seek(0) key = default_storage.save(key, file) return http.HttpResponse( diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6f60592..0000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length=88 -select = C,E,F,W,B,B950 -ignore = E203, E501, W503, E731 diff --git a/tests/conftest.py b/tests/conftest.py index 54f7de2..8bf1fa1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,5 +78,5 @@ def filemodel(request, db): from tests.testapp.models import FileModel return FileModel.objects.create( - file=ContentFile(request.node.name, "%s.txt" % request.node.name) + file=ContentFile(request.node.name, f"{request.node.name}.txt") ) diff --git a/tests/test_forms.py b/tests/test_forms.py index c2d33ba..df4c579 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -200,7 +200,7 @@ def test_file_insert( assert file_input.get_attribute("name") == "file" with wait_for_page_load(driver, timeout=10): file_input.submit() - assert storage.exists("tmp/s3file/%s.txt" % request.node.name) + assert storage.exists(f"tmp/s3file/{request.node.name}.txt") with pytest.raises(NoSuchElementException): error = driver.find_element(By.XPATH, "//body[@JSError]") @@ -221,7 +221,7 @@ def test_file_update( assert file_input.get_attribute("name") == "file" with wait_for_page_load(driver, timeout=10): file_input.submit() - assert storage.exists("tmp/s3file/%s.txt" % request.node.name) + assert storage.exists(f"tmp/s3file/{request.node.name}.txt") with pytest.raises(NoSuchElementException): error = driver.find_element(By.XPATH, "//body[@JSError]") @@ -288,12 +288,10 @@ def test_multi_file( driver.get(live_server + reverse_lazy("upload-multi")) file_input = driver.find_element(By.XPATH, "//input[@name='file']") file_input.send_keys( - " \n ".join( - [ - str(freeze_upload_folder / upload_file), - str(freeze_upload_folder / another_upload_file), - ] - ) + " \n ".join([ + str(freeze_upload_folder / upload_file), + str(freeze_upload_folder / another_upload_file), + ]) ) file_input = driver.find_element(By.XPATH, "//input[@name='other_file']") file_input.send_keys(str(freeze_upload_folder / yet_another_upload_file)) diff --git a/tests/testapp/urls.py b/tests/testapp/urls.py index 9f54c24..b37b04c 100644 --- a/tests/testapp/urls.py +++ b/tests/testapp/urls.py @@ -8,18 +8,14 @@ urlpatterns = [ path( "example/", - include( - [ - path( - "create", views.ExampleCreateView.as_view(), name="example-create" - ), - path( - "/update", - views.ExampleUpdateView.as_view(), - name="example-update", - ), - ] - ), + include([ + path("create", views.ExampleCreateView.as_view(), name="example-create"), + path( + "/update", + views.ExampleUpdateView.as_view(), + name="example-update", + ), + ]), ), path("multi/", views.MultiExampleFormView.as_view(), name="upload-multi"), ]