bpo-36786: Run compileall in parallel during "make install" (pythonGH-13078)

Author: pitrou

Doc/library/compileall.rst

@@ -158,7 +158,8 @@ Public functions
    The argument *workers* specifies how many workers are used to
    compile files in parallel. The default is to not use multiple workers.
    If the platform can't use multiple workers and *workers* argument is given,
-   then sequential compilation will be used as a fallback.  If *workers* is
+   then sequential compilation will be used as a fallback.  If *workers*
+   is 0, the number of cores in the system is used.  If *workers* is
    lower than ``0``, a :exc:`ValueError` will be raised.
 
    *invalidation_mode* should be a member of the
@@ -184,6 +185,9 @@ Public functions
    .. versionchanged:: 3.7
       The *invalidation_mode* parameter was added.
 
+   .. versionchanged:: 3.8
+      Setting *workers* to 0 now chooses the optimal number of cores.
+
 .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP)
 
    Compile the file with path *fullname*. Return a true value if the file

Lib/compileall.py

@@ -67,20 +67,20 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
     invalidation_mode: how the up-to-dateness of the pyc will be checked
     """
     ProcessPoolExecutor = None
-    if workers is not None:
-        if workers < 0:
-            raise ValueError('workers must be greater or equal to 0')
-        elif workers != 1:
-            try:
-                # Only import when needed, as low resource platforms may
-                # fail to import it
-                from concurrent.futures import ProcessPoolExecutor
-            except ImportError:
-                workers = 1
+    if workers < 0:
+        raise ValueError('workers must be greater or equal to 0')
+    if workers != 1:
+        try:
+            # Only import when needed, as low resource platforms may
+            # fail to import it
+            from concurrent.futures import ProcessPoolExecutor
+        except ImportError:
+            workers = 1
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
                       ddir=ddir)
     success = True
-    if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
+    if workers != 1 and ProcessPoolExecutor is not None:
+        # If workers == 0, let ProcessPoolExecutor choose
         workers = workers or None
         with ProcessPoolExecutor(max_workers=workers) as executor:
             results = executor.map(partial(compile_file,
@@ -290,9 +290,6 @@ def main():
                 print("Error reading file list {}".format(args.flist))
             return False
 
-    if args.workers is not None:
-        args.workers = args.workers or None
-
     if args.invalidation_mode:
         ivl_mode = args.invalidation_mode.replace('-', '_').upper()
         invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]

Lib/test/test_compileall.py

@@ -575,7 +575,7 @@ def test_workers_available_cores(self, compile_dir):
                         new=[sys.executable, self.directory, "-j0"]):
             compileall.main()
             self.assertTrue(compile_dir.called)
-            self.assertEqual(compile_dir.call_args[-1]['workers'], None)
+            self.assertEqual(compile_dir.call_args[-1]['workers'], 0)
 
 
 class CommmandLineTestsWithSourceEpoch(CommandLineTestsBase,

Makefile.pre.in

@@ -1432,30 +1432,30 @@ libinstall:	build_all $(srcdir)/Modules/xxmodule.c
 	fi
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST)  $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST) -f \
+		-j0 -d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST) -f \
+		-j0 -d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -OO $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST) -f \
+		-j0 -d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST)/site-packages -f \
+		-j0 -d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST)/site-packages -f \
+		-j0 -d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -OO $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST)/site-packages -f \
+		-j0 -d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt

Misc/NEWS.d/next/Build/2019-05-03-21-08-06.bpo-36786.gOLFbD.rst

@@ -0,0 +1 @@
+"make install" now runs compileall in parallel.