Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
bpo-29877: compileall: import ProcessPoolExecutor only when needed (G…
…H-4856)

Importing ProcessPoolExecutor may hang or cause an error when the import
accesses urandom on a low resource platform


https://bugs.python.org/issue29877
  • Loading branch information
virtuald authored and miss-islington committed Nov 23, 2018
1 parent 9de3632 commit 1d817e4
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 11 deletions.
18 changes: 11 additions & 7 deletions Lib/compileall.py
Expand Up @@ -16,10 +16,6 @@
import py_compile
import struct

try:
from concurrent.futures import ProcessPoolExecutor
except ImportError:
ProcessPoolExecutor = None
from functools import partial

__all__ = ["compile_dir","compile_file","compile_path"]
Expand Down Expand Up @@ -70,9 +66,17 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
workers: maximum number of parallel workers
invalidation_mode: how the up-to-dateness of the pyc will be checked
"""
if workers is not None and workers < 0:
raise ValueError('workers must be greater or equal to 0')

ProcessPoolExecutor = None
if workers is not None:
if workers < 0:
raise ValueError('workers must be greater or equal to 0')
elif workers != 1:
try:
# Only import when needed, as low resource platforms may
# fail to import it
from concurrent.futures import ProcessPoolExecutor
except ImportError:
workers = 1
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
ddir=ddir)
success = True
Expand Down
8 changes: 4 additions & 4 deletions Lib/test/test_compileall.py
Expand Up @@ -167,7 +167,7 @@ def test_compile_dir_pathlike(self):
self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)')
self.assertTrue(os.path.isfile(self.bc_path))

@mock.patch('compileall.ProcessPoolExecutor')
@mock.patch('concurrent.futures.ProcessPoolExecutor')
def test_compile_pool_called(self, pool_mock):
compileall.compile_dir(self.directory, quiet=True, workers=5)
self.assertTrue(pool_mock.called)
Expand All @@ -177,19 +177,19 @@ def test_compile_workers_non_positive(self):
"workers must be greater or equal to 0"):
compileall.compile_dir(self.directory, workers=-1)

@mock.patch('compileall.ProcessPoolExecutor')
@mock.patch('concurrent.futures.ProcessPoolExecutor')
def test_compile_workers_cpu_count(self, pool_mock):
compileall.compile_dir(self.directory, quiet=True, workers=0)
self.assertEqual(pool_mock.call_args[1]['max_workers'], None)

@mock.patch('compileall.ProcessPoolExecutor')
@mock.patch('concurrent.futures.ProcessPoolExecutor')
@mock.patch('compileall.compile_file')
def test_compile_one_worker(self, compile_file_mock, pool_mock):
compileall.compile_dir(self.directory, quiet=True)
self.assertFalse(pool_mock.called)
self.assertTrue(compile_file_mock.called)

@mock.patch('compileall.ProcessPoolExecutor', new=None)
@mock.patch('concurrent.futures.ProcessPoolExecutor', new=None)
@mock.patch('compileall.compile_file')
def test_compile_missing_multiprocessing(self, compile_file_mock):
compileall.compile_dir(self.directory, quiet=True, workers=5)
Expand Down
@@ -0,0 +1,2 @@
compileall: import ProcessPoolExecutor only when needed, preventing hangs on
low resource platforms

0 comments on commit 1d817e4

Please sign in to comment.