Add merge-wait FEATURES setting enabled by default

Add a new merge-wait FEATURES setting, enabled by default,
which prevents packages from being merged while other packages
are building. Previously, this behavior was already implemented
for packages that satisfy direct or indirect dependencies of
the system set. This feature can be disabled only for other
packages which do not satisfy direct or indirect dependencies
of the system set, in order to trade the possibility of random
build failures for greater parallelism.

Currently, it is known that having merge-wait disabled can
cause "random" build failures for builds using setuptools when
setuptools plugins are merged during the build.

Bug: https://bugs.gentoo.org/663324
Signed-off-by: Zac Medico <zmedico@gentoo.org>

cnf/make.globals

@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 # System-wide defaults for the Portage system
 
@@ -77,7 +77,7 @@ FETCHCOMMAND_SFTP="bash -c \"x=\\\${2#sftp://} ; host=\\\${x%%/*} ; port=\\\${ho
 FEATURES="assume-digests binpkg-docompress binpkg-dostrip binpkg-logs
           binpkg-multi-instance buildpkg-live
           config-protect-if-modified distlocks ebuild-locks
-          fixlafiles ipc-sandbox merge-sync multilib-strict
+          fixlafiles ipc-sandbox merge-sync merge-wait multilib-strict
           network-sandbox news parallel-fetch pkgdir-index-trusted pid-sandbox
           preserve-libs protect-owned qa-unresolved-soname-deps
           sandbox sfperms strict

lib/_emerge/Scheduler.py

@@ -1521,14 +1521,16 @@ def _build_exit(self, build):
             self.curval += 1
             merge = PackageMerge(merge=build, scheduler=self._sched_iface)
             self._running_tasks[id(merge)] = merge
-            if (
-                not build.build_opts.buildpkgonly
-                and build.pkg in self._deep_system_deps
+            # By default, merge-wait only allows merge when no builds are executing.
+            # As a special exception, dependencies on system packages are frequently
+            # unspecified and will therefore force merge-wait.
+            is_system_pkg = build.pkg in self._deep_system_deps
+            if not build.build_opts.buildpkgonly and (
+                "merge-wait" in build.settings.features or is_system_pkg
             ):
-                # Since dependencies on system packages are frequently
-                # unspecified, merge them only when no builds are executing.
                 self._merge_wait_queue.append(merge)
-                merge.addStartListener(self._system_merge_started)
+                if is_system_pkg:
+                    merge.addStartListener(self._system_merge_started)
             else:
                 self._task_queues.merge.add(merge)
                 merge.addExitListener(self._merge_exit)

lib/portage/const.py

@@ -1,5 +1,5 @@
 # portage: Constants
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import os
@@ -194,6 +194,7 @@
         "keepwork",
         "lmirror",
         "merge-sync",
+        "merge-wait",
         "metadata-transfer",
         "mirror",
         "mount-sandbox",

man/make.conf.5

@@ -1,4 +1,4 @@
-.TH "MAKE.CONF" "5" "Mar 2023" "Portage @VERSION@" "Portage"
+.TH "MAKE.CONF" "5" "Jan 2024" "Portage @VERSION@" "Portage"
 .SH "NAME"
 make.conf \- custom settings for Portage
 .SH "SYNOPSIS"
@@ -565,6 +565,15 @@ After a package is merged or unmerged, sync relevant files to
 disk in order to avoid data\-loss in the event of a power failure.
 This feature is enabled by default.
 .TP
+.B merge\-wait
+Wait for all builds to complete before merging new packages, which only
+matters when using the \fBemerge\fR(1) \fB\-\-jobs\fR option. This
+feature is always effectively enabled and it cannot be disabled for
+packages that satisfy direct or indirect dependencies of the system
+set. For other packages, it can be disabled in order to trade the
+possibility of random build failures for greater parallelism.
+This feature is enabled by default.
+.TP
 .B metadata\-transfer
 Automatically perform a metadata transfer when `emerge \-\-sync` is run.
 In versions of portage >=2.1.5, this feature is disabled by
@@ -637,8 +646,9 @@ terminal to view parallel-fetch progress.
 .TP
 .B parallel\-install
 Use finer\-grained locks when installing packages, allowing for greater
-parallelization. For additional parallelization, disable
-\fIebuild\-locks\fR.
+parallelism. Note that \fIparallel\-install\fR currently has no effect
+unless \fImerge\-wait\fR is disabled. For additional parallelism,
+disable \fIebuild\-locks\fR.
 .TP
 .B pid\-sandbox
 Isolate the process space for the ebuild processes. This makes it