Restrict pylint parallelism to available RAM

pylint uses *lots* of RAM (more than 1 GiB per process), and the
`jobs=0` default runs `nproc` parallel processes by default. This
exceeds the available memory in many standard environments (such as
m1.medium == 2 CPUs/4 GiB RAM, or m1.large == 4 CPUs/8 GiB RAM) and
causes eternal hangs of the pylintrc test when one job process gets
OOM-killed.

Instead, set the parallelism depending on the amount of free memory --
one job for each 2 GB of available RAM. This is an underestimation, but
let's not use up the entire system memory. Also still cap it to the
number of available CPUs, just in case this runs on a system with lots
of RAM and only few CPUs.

Cherry-picked from master commit be12ca7

Related: rhbz#1885635

tests/pylint/censorship.py

@@ -32,6 +32,7 @@
 
 import sys
 import re
+import multiprocessing
 
 from io import StringIO
 
@@ -115,6 +116,23 @@ def run(self):
     def _prepare_args(self):
         args = []
 
+        # pylint uses a lot of memory and doesn't handle ENOMEM well, so set --jobs based on memory
+        # do this early so that command_line_args can override -j
+        avail_mem_kb = 0
+        with open("/proc/meminfo") as f:
+            for line in f:
+                if line.startswith("MemAvailable:"):
+                    avail_mem_kb = int(line.split()[1])
+                    break
+        num_cpus = multiprocessing.cpu_count()
+        # each process uses ~ 1.5 GiB RAM, leave some breathing space
+        jobs = min(1, avail_mem_kb // 2000000)
+        # but also clip to nproc
+        jobs = max(jobs, num_cpus)
+        print("Using", jobs, "parallel jobs based on", avail_mem_kb, "kB available RAM and",
+              num_cpus, "CPUs")
+        args.append("-j%i" % jobs)
+
         if self._config.command_line_args:
             args = self._config.command_line_args
 

tests/pylint/pylintrc

@@ -17,10 +17,6 @@ ignore-patterns=
 # pygtk.require().
 init-hook='import gi.overrides, os; gi.overrides.__path__[0:0] = (os.environ["ANACONDA_WIDGETS_OVERRIDES"].split(":") if "ANACONDA_WIDGETS_OVERRIDES" in os.environ else [])'
 
-# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
-# number of processors available to use.
-jobs=0
-
 # Control the amount of potential inferred values when inferring a single
 # object. This can help the performance when dealing with large functions or
 # complex, nested conditions.