DEV: use number of physical cores in dev.py build by default (scipy#18451)

Author: rgommers

.github/workflows/linux_meson.yml

@@ -93,7 +93,7 @@ jobs:
 
     - name: Setup build and install scipy
       run: |
-        python dev.py build -j 2 --werror
+        python dev.py build --werror
 
     - name: Ccache performance
       shell: bash -l {0}
@@ -119,7 +119,7 @@ jobs:
       run: |
         export OMP_NUM_THREADS=2
         export SCIPY_USE_PROPACK=1
-        python dev.py --no-build test -j 2 -- --durations 10 --timeout=60
+        python dev.py --no-build test -j2 -- --durations 10 --timeout=60
 
   #################################################################################
   test_venv_install:
@@ -317,7 +317,7 @@ jobs:
 
     - name: Build and install SciPy
       run: |
-        python dev.py build -j2 --gcov
+        python dev.py build --gcov
 
     - name: Ccache performance
       shell: bash -l {0}

.github/workflows/macos_meson.yml

@@ -113,15 +113,15 @@ jobs:
         # https://conda-forge.org/docs/maintainer/knowledge_base.html#requiring-newer-macos-sdks
         export MACOSX_DEPLOYMENT_TARGET=10.9
         export MACOSX_SDK_VERSION=10.9
-        CC="ccache $CC" python dev.py build -j 2
+        CC="ccache $CC" python dev.py build
 
     - name: Test SciPy
       shell: bash -l {0}
       run: |
         conda activate scipy-dev
         export OMP_NUM_THREADS=2
         export SCIPY_USE_PROPACK=1
-        python dev.py -n test -j 2
+        python dev.py -n test -j2
 
     - name: Ccache statistics
       shell: bash -l {0}

.github/workflows/windows.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Build
         run: |
           echo "SCIPY_USE_PROPACK=1" >> $env:GITHUB_ENV
-          python dev.py build -j 2 --win-cp-openblas
+          python dev.py build --win-cp-openblas
           # Necessary because GitHub Actions checks out the repo to D:\ while OpenBLAS
           # got installed to C:\ higher up. The copying with `--win-cp-openblas` fails
           # when things are split over drives.
@@ -105,7 +105,7 @@ jobs:
 
       - name: Build
         run: |
-          python dev.py build -j 2 --win-cp-openblas
+          python dev.py build --win-cp-openblas
           # Copy OpenBLAS DLL, write distributor-init (see first job in this file for why)
           cp C:\opt\64\bin\*.dll $pwd\build-install\Lib\site-packages\scipy\.libs\
           python tools\openblas_support.py --write-init $PWD\build-install\Lib\site-packages\scipy\

dev.py

@@ -107,6 +107,9 @@ def task_meta(cls, **kwargs):
 import errno
 import contextlib
 from sysconfig import get_path
+import math
+import traceback
+from concurrent.futures.process import _MAX_WINDOWS_WORKERS
 
 # distutils is required to infer meson install path
 # if this needs to be replaced for Python 3.12 support and there's no
@@ -414,7 +417,7 @@ class Build(Task):
     parallel = Option(
         ['--parallel', '-j'], default=None, metavar='N_JOBS',
         help=("Number of parallel jobs for building. "
-              "This defaults to 2 * n_cpus + 2."))
+              "This defaults to the number of available physical CPU cores"))
     setup_args = Option(
         ['--setup-args', '-C'], default=[], multiple=True,
         help=("Pass along one or more arguments to `meson setup` "
@@ -499,7 +502,12 @@ def build_project(cls, dirs, args, env):
         Build a dev version of the project.
         """
         cmd = ["ninja", "-C", str(dirs.build)]
-        if args.parallel is not None:
+        if args.parallel is None:
+            # Use number of physical cores rather than ninja's default of 2N+2,
+            # to avoid out of memory issues (see gh-17941 and gh-18443)
+            n_cores = cpu_count(only_physical_cores=True)
+            cmd += [f"-j{n_cores}"]
+        else:
             cmd += ["-j", str(args.parallel)]
 
         # Building with ninja-backend
@@ -1226,5 +1234,224 @@ def authors(ctx_obj, revision_args):
         print('Error caught: Incorrect revision start or revision end')
 
 
+# The following CPU core count functions were taken from loky/backend/context.py
+# See https://github.com/joblib/loky
+
+# Cache for the number of physical cores to avoid repeating subprocess calls.
+# It should not change during the lifetime of the program.
+physical_cores_cache = None
+
+
+def cpu_count(only_physical_cores=False):
+    """Return the number of CPUs the current process can use.
+
+    The returned number of CPUs accounts for:
+     * the number of CPUs in the system, as given by
+       ``multiprocessing.cpu_count``;
+     * the CPU affinity settings of the current process
+       (available on some Unix systems);
+     * Cgroup CPU bandwidth limit (available on Linux only, typically
+       set by docker and similar container orchestration systems);
+     * the value of the LOKY_MAX_CPU_COUNT environment variable if defined.
+    and is given as the minimum of these constraints.
+
+    If ``only_physical_cores`` is True, return the number of physical cores
+    instead of the number of logical cores (hyperthreading / SMT). Note that
+    this option is not enforced if the number of usable cores is controlled in
+    any other way such as: process affinity, Cgroup restricted CPU bandwidth
+    or the LOKY_MAX_CPU_COUNT environment variable. If the number of physical
+    cores is not found, return the number of logical cores.
+
+    Note that on Windows, the returned number of CPUs cannot exceed 61 (or 60 for
+    Python < 3.10), see:
+    https://bugs.python.org/issue26903.
+
+    It is also always larger or equal to 1.
+    """
+    # Note: os.cpu_count() is allowed to return None in its docstring
+    os_cpu_count = os.cpu_count() or 1
+    if sys.platform == "win32":
+        # On Windows, attempting to use more than 61 CPUs would result in a
+        # OS-level error. See https://bugs.python.org/issue26903. According to
+        # https://learn.microsoft.com/en-us/windows/win32/procthread/processor-groups
+        # it might be possible to go beyond with a lot of extra work but this
+        # does not look easy.
+        os_cpu_count = min(os_cpu_count, _MAX_WINDOWS_WORKERS)
+
+    cpu_count_user = _cpu_count_user(os_cpu_count)
+    aggregate_cpu_count = max(min(os_cpu_count, cpu_count_user), 1)
+
+    if not only_physical_cores:
+        return aggregate_cpu_count
+
+    if cpu_count_user < os_cpu_count:
+        # Respect user setting
+        return max(cpu_count_user, 1)
+
+    cpu_count_physical, exception = _count_physical_cores()
+    if cpu_count_physical != "not found":
+        return cpu_count_physical
+
+    # Fallback to default behavior
+    if exception is not None:
+        # warns only the first time
+        warnings.warn(
+            "Could not find the number of physical cores for the "
+            f"following reason:\n{exception}\n"
+            "Returning the number of logical cores instead. You can "
+            "silence this warning by setting LOKY_MAX_CPU_COUNT to "
+            "the number of cores you want to use."
+        )
+        traceback.print_tb(exception.__traceback__)
+
+    return aggregate_cpu_count
+
+
+def _cpu_count_cgroup(os_cpu_count):
+    # Cgroup CPU bandwidth limit available in Linux since 2.6 kernel
+    cpu_max_fname = "/sys/fs/cgroup/cpu.max"
+    cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
+    cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
+    if os.path.exists(cpu_max_fname):
+        # cgroup v2
+        # https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
+        with open(cpu_max_fname) as fh:
+            cpu_quota_us, cpu_period_us = fh.read().strip().split()
+    elif os.path.exists(cfs_quota_fname) and os.path.exists(cfs_period_fname):
+        # cgroup v1
+        # https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management
+        with open(cfs_quota_fname) as fh:
+            cpu_quota_us = fh.read().strip()
+        with open(cfs_period_fname) as fh:
+            cpu_period_us = fh.read().strip()
+    else:
+        # No Cgroup CPU bandwidth limit (e.g. non-Linux platform)
+        cpu_quota_us = "max"
+        cpu_period_us = 100_000  # unused, for consistency with default values
+
+    if cpu_quota_us == "max":
+        # No active Cgroup quota on a Cgroup-capable platform
+        return os_cpu_count
+    else:
+        cpu_quota_us = int(cpu_quota_us)
+        cpu_period_us = int(cpu_period_us)
+        if cpu_quota_us > 0 and cpu_period_us > 0:
+            return math.ceil(cpu_quota_us / cpu_period_us)
+        else:  # pragma: no cover
+            # Setting a negative cpu_quota_us value is a valid way to disable
+            # cgroup CPU bandwith limits
+            return os_cpu_count
+
+
+def _cpu_count_affinity(os_cpu_count):
+    # Number of available CPUs given affinity settings
+    if hasattr(os, "sched_getaffinity"):
+        try:
+            return len(os.sched_getaffinity(0))
+        except NotImplementedError:
+            pass
+
+    # On PyPy and possibly other platforms, os.sched_getaffinity does not exist
+    # or raises NotImplementedError, let's try with the psutil if installed.
+    try:
+        import psutil
+
+        p = psutil.Process()
+        if hasattr(p, "cpu_affinity"):
+            return len(p.cpu_affinity())
+
+    except ImportError:  # pragma: no cover
+        if (
+            sys.platform == "linux"
+            and os.environ.get("LOKY_MAX_CPU_COUNT") is None
+        ):
+            # PyPy does not implement os.sched_getaffinity on Linux which
+            # can cause severe oversubscription problems. Better warn the
+            # user in this particularly pathological case which can wreck
+            # havoc, typically on CI workers.
+            warnings.warn(
+                "Failed to inspect CPU affinity constraints on this system. "
+                "Please install psutil or explictly set LOKY_MAX_CPU_COUNT."
+            )
+
+    # This can happen for platforms that do not implement any kind of CPU
+    # infinity such as macOS-based platforms.
+    return os_cpu_count
+
+
+def _cpu_count_user(os_cpu_count):
+    """Number of user defined available CPUs"""
+    cpu_count_affinity = _cpu_count_affinity(os_cpu_count)
+
+    cpu_count_cgroup = _cpu_count_cgroup(os_cpu_count)
+
+    # User defined soft-limit passed as a loky specific environment variable.
+    cpu_count_loky = int(os.environ.get("LOKY_MAX_CPU_COUNT", os_cpu_count))
+
+    return min(cpu_count_affinity, cpu_count_cgroup, cpu_count_loky)
+
+
+def _count_physical_cores():
+    """Return a tuple (number of physical cores, exception)
+
+    If the number of physical cores is found, exception is set to None.
+    If it has not been found, return ("not found", exception).
+
+    The number of physical cores is cached to avoid repeating subprocess calls.
+    """
+    exception = None
+
+    # First check if the value is cached
+    global physical_cores_cache
+    if physical_cores_cache is not None:
+        return physical_cores_cache, exception
+
+    # Not cached yet, find it
+    try:
+        if sys.platform == "linux":
+            cpu_info = subprocess.run(
+                "lscpu --parse=core".split(), capture_output=True, text=True
+            )
+            cpu_info = cpu_info.stdout.splitlines()
+            cpu_info = {line for line in cpu_info if not line.startswith("#")}
+            cpu_count_physical = len(cpu_info)
+        elif sys.platform == "win32":
+            cpu_info = subprocess.run(
+                "wmic CPU Get NumberOfCores /Format:csv".split(),
+                capture_output=True,
+                text=True,
+            )
+            cpu_info = cpu_info.stdout.splitlines()
+            cpu_info = [
+                l.split(",")[1]
+                for l in cpu_info
+                if (l and l != "Node,NumberOfCores")
+            ]
+            cpu_count_physical = sum(map(int, cpu_info))
+        elif sys.platform == "darwin":
+            cpu_info = subprocess.run(
+                "sysctl -n hw.physicalcpu".split(),
+                capture_output=True,
+                text=True,
+            )
+            cpu_info = cpu_info.stdout
+            cpu_count_physical = int(cpu_info)
+        else:
+            raise NotImplementedError(f"unsupported platform: {sys.platform}")
+
+        # if cpu_count_physical < 1, we did not find a valid value
+        if cpu_count_physical < 1:
+            raise ValueError(f"found {cpu_count_physical} physical cores < 1")
+
+    except Exception as e:
+        exception = e
+        cpu_count_physical = "not found"
+
+    # Put the result in cache
+    physical_cores_cache = cpu_count_physical
+
+    return cpu_count_physical, exception
+
+
 if __name__ == '__main__':
     cli()