diff --git a/.travis.yml b/.travis.yml index 73ed1f3..6dbd5e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ env: matrix: allow_failures: - env: DJANGO="<1.7,>=1.6" + - python: "pypy" + - python: "pyp3" install: - pip install -U pip - pip install . diff --git a/README.rst b/README.rst index 465a967..9306f17 100644 --- a/README.rst +++ b/README.rst @@ -214,6 +214,9 @@ Multi processing It not only increased performance but the garbage collection and therefore the huge memory footprint from previous versions. + **Note:** PyPy seems to have some problems regarding multiprocessing, + for that matter all multiprocessing is disabled in PyPy. + Testing ------- diff --git a/pytest.ini b/pytest.ini index be372c8..a34777f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -norecursedirs=venv +norecursedirs=venv env DJANGO_SETTINGS_MODULE=tests.settings addopts = --tb=short --pep8 --isort --flakes -rxs pep8ignore= diff --git a/stdimage/management/commands/rendervariations.py b/stdimage/management/commands/rendervariations.py index d0d8318..e9855dc 100644 --- a/stdimage/management/commands/rendervariations.py +++ b/stdimage/management/commands/rendervariations.py @@ -12,6 +12,7 @@ from stdimage.utils import render_variations +is_pypy = '__pypy__' in sys.builtin_module_names BAR = None @@ -43,24 +44,47 @@ def handle(self, *args, **options): queryset = model_class._default_manager \ .exclude(**{'%s__isnull' % field_name: True}) \ .exclude(**{field_name: ''}) - images = queryset.values_list(field_name, flat=True) - - pool = Pool( - initializer=init_progressbar, - initargs=[queryset.count()] + images = queryset.values_list(field_name, flat=True).iterator() + count = queryset.count() + + if is_pypy: # pypy doesn't handle multiprocessing to well + self.render_linear(field, images, count, replace) + else: + self.render_in_parallel(field, images, count, replace) + + @staticmethod + def render_in_parallel(field, images, count, replace): + pool = Pool( + initializer=init_progressbar, + initargs=[count] + ) + args = [ + dict( + file_name=file_name, + variations=field.variations, + replace=replace, + ) + for file_name in images + ] + pool.map(render_field_variations, args) + pool.apply(finish_progressbar) + pool.close() + pool.join() + + @staticmethod + def render_linear(field, images, count, replace): + init_progressbar(count) + args_list = [ + dict( + file_name=file_name, + variations=field.variations, + replace=replace, ) - args = [ - dict( - file_name=file_name, - variations=field.variations, - replace=replace, - ) - for file_name in images - ] - pool.map(render_field_variations, args) - pool.apply(finish_progressbar) - pool.close() - pool.join() + for file_name in images + ] + for args in args_list: + render_variations(**args) + finish_progressbar() def init_progressbar(count):