From d2f74ca27107b71677edac81be45a8fe8a184a95 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 4 Mar 2021 18:28:21 +0100 Subject: [PATCH 1/8] Drop multiprocessing in rendervariations command It is profered to use asynchronous renders andyways and to do image rendering on optimized works. Therefore, using multiprocessing to simply queue tasks adds more errors than it does good. --- README.md | 10 ---------- stdimage/management/commands/rendervariations.py | 9 ++------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 709b586..e646511 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ Django Field that implement the following features: * Access thumbnails on model level, no template tags required * Preserves original image * Asynchronous rendering (Celery & Co) -* Multi threading and processing for optimum performance * Restrict accepted image dimensions * Rename files to a standardized name (using a callable upload_to) @@ -195,12 +194,3 @@ The `replace` option will replace all existing files. The `ignore-missing` option will suspend missing source file errors and keep rendering variations for other files. Othervise command will stop on first missing file. - -### Multi processing -Since version 2 stdImage supports multiprocessing. -Every image is rendered in separate process. -It not only increased performance but the garbage collection -and therefore the huge memory footprint from previous versions. - -**Note:** PyPy seems to have some problems regarding multiprocessing, -for that matter all multiprocessing is disabled in PyPy. diff --git a/stdimage/management/commands/rendervariations.py b/stdimage/management/commands/rendervariations.py index 86a0a42..2ba3df3 100644 --- a/stdimage/management/commands/rendervariations.py +++ b/stdimage/management/commands/rendervariations.py @@ -1,6 +1,3 @@ -from concurrent.futures import ProcessPoolExecutor -from multiprocessing import cpu_count - import progressbar from django.apps import apps from django.core.files.storage import get_storage_class @@ -75,14 +72,12 @@ def render(field, images, count, replace, ignore_missing, do_render): ) with progressbar.ProgressBar(max_value=count, widgets=( progressbar.RotatingMarker(), - ' | CPUs: {}'.format(cpu_count()), ' | ', progressbar.AdaptiveETA(), ' | ', progressbar.Percentage(), ' ', progressbar.Bar(), )) as bar: - with ProcessPoolExecutor() as executor: - for _ in executor.map(render_field_variations, kwargs_list): - bar += 1 + for _ in map(render_field_variations, kwargs_list): + bar += 1 def render_field_variations(kwargs): From fe53def186308d1209f1151d92cba7665233583f Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 4 Mar 2021 18:30:01 +0100 Subject: [PATCH 2/8] Fix bandit config --- .bandit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit b/.bandit index fba0d25..b7df176 100644 --- a/.bandit +++ b/.bandit @@ -1,2 +1,2 @@ [bandit] -exclude: tests +exclude: ./tests From ab63a991737d0bb4aef1697ccf12e07079096c9a Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 4 Mar 2021 18:31:53 +0100 Subject: [PATCH 3/8] Add dependabot config --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..71607d0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily From 62ea2b2818aea27f5bbb354dc07ca6224636c051 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 4 Mar 2021 18:32:04 +0100 Subject: [PATCH 4/8] Unfreeze pytest test dependency --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 162dda1..766944a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ setup_requires = setuptools_scm pytest-runner tests_require = - pytest>=4.0,<5.0 + pytest pytest-cov pytest-django From b47bce9fc14ef82a33033d10c98eb083821f1d44 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 4 Mar 2021 18:34:24 +0100 Subject: [PATCH 5/8] Update Django and Python version support --- .github/workflows/tests.yml | 11 ++++++++--- setup.cfg | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17f8f14..c2168dc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,8 +9,13 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.5, 3.6, 3.7] - django-version: [1.11.*, 2.2.*] + python-version: + - "3.7" + - "3.8" + - "3.9" + django-version: + - "2.2" + - "3.1" steps: - uses: actions/checkout@v1 @@ -21,7 +26,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install django==${{ matrix.django-version }} + pip install django~=${{ matrix.django-version }} - name: Test with pytest run: python setup.py test - name: Codecov diff --git a/setup.cfg b/setup.cfg index 766944a..4255691 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,7 @@ classifier = include_package_data = True packages = stdimage install_requires = - Django>=1.11 + Django>=2.2 pillow>=2.5 progressbar2>=3.0.0 setup_requires = From 233b434f59dfa37c8772f26b42a63f298e8484fa Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 4 Mar 2021 18:35:33 +0100 Subject: [PATCH 6/8] Switch to codecov Python package --- .github/workflows/tests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c2168dc..a3cec88 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,11 +25,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools + python -m pip install --upgrade pip setuptools codecov pip install django~=${{ matrix.django-version }} - name: Test with pytest run: python setup.py test - name: Codecov - uses: codecov/codecov-action@v1.0.2 - with: - token: ${{secrets.CODECOV_TOKEN}} + run: codecov From 8e621f9404699d4f2e7905ed8d18878a88522b9d Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 4 Mar 2021 18:38:09 +0100 Subject: [PATCH 7/8] Add dist check step --- .github/workflows/tests.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a3cec88..7c67ce9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,10 +1,25 @@ name: Tests -on: [push] +on: + push: + branches: + - master + pull_request: jobs: - build: + dist: + runs-on: ubuntu-latest + steps: + - name: Install gettext + run: sudo apt-get install gettext -y + - uses: actions/setup-python@v2 + - run: python -m pip install --upgrade pip setuptools wheel twine readme-renderer + - uses: actions/checkout@v2 + - run: python setup.py sdist bdist_wheel + - run: python -m twine check dist/* + + pytest: runs-on: ubuntu-latest strategy: max-parallel: 4 @@ -16,7 +31,6 @@ jobs: django-version: - "2.2" - "3.1" - steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} From c86bd33099651b7fd936eafb065e4d4929e6dbc2 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 4 Mar 2021 18:40:04 +0100 Subject: [PATCH 8/8] Simplify CI setup --- .github/workflows/tests.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7c67ce9..026496f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: Tests +name: CI on: push: @@ -22,7 +22,6 @@ jobs: pytest: runs-on: ubuntu-latest strategy: - max-parallel: 4 matrix: python-version: - "3.7" @@ -32,16 +31,15 @@ jobs: - "2.2" - "3.1" steps: - - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v1 - name: Install dependencies run: | python -m pip install --upgrade pip setuptools codecov pip install django~=${{ matrix.django-version }} - name: Test with pytest run: python setup.py test - - name: Codecov - run: codecov + - run: codecov