Merge pull request #2918 from martinpitt/pylint-jobs

Restrict pylint parallelism to available RAM

tests/pylint/censorship.py

@@ -32,6 +32,7 @@
 
 import sys
 import re
+import multiprocessing
 
 from io import StringIO
 
@@ -116,6 +117,23 @@ def run(self):
     def _prepare_args(self):
         args = []
 
+        # pylint uses a lot of memory and doesn't handle ENOMEM well, so set --jobs based on memory
+        # do this early so that command_line_args can override -j
+        avail_mem_kb = 0
+        with open("/proc/meminfo") as f:
+            for line in f:
+                if line.startswith("MemAvailable:"):
+                    avail_mem_kb = int(line.split()[1])
+                    break
+        num_cpus = multiprocessing.cpu_count()
+        # each process uses ~ 1.5 GiB RAM, leave some breathing space
+        jobs = min(1, avail_mem_kb // 2000000)
+        # but also clip to nproc
+        jobs = max(jobs, num_cpus)
+        print("Using", jobs, "parallel jobs based on", avail_mem_kb, "kB available RAM and",
+              num_cpus, "CPUs")
+        args.append("-j%i" % jobs)
+
         if self._config.command_line_args:
             args = self._config.command_line_args
 

tests/pylint/pylintrc

@@ -17,10 +17,6 @@ ignore-patterns=
 # pygtk.require().
 init-hook='import gi.overrides, os; gi.overrides.__path__[0:0] = (os.environ["ANACONDA_WIDGETS_OVERRIDES"].split(":") if "ANACONDA_WIDGETS_OVERRIDES" in os.environ else [])'
 
-# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
-# number of processors available to use.
-jobs=0
-
 # Control the amount of potential inferred values when inferring a single
 # object. This can help the performance when dealing with large functions or
 # complex, nested conditions.