Scheduler: Support parallel-install with merge-wait

lib/_emerge/PackageMerge.py

@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from _emerge.CompositeTask import CompositeTask
@@ -7,7 +7,7 @@
 
 
 class PackageMerge(CompositeTask):
-    __slots__ = ("merge", "postinst_failure")
+    __slots__ = ("is_system_pkg", "merge", "postinst_failure")
 
     def _should_show_status(self):
         return (

lib/_emerge/Scheduler.py

@@ -1519,17 +1519,20 @@ def _build_exit(self, build):
             self._deallocate_config(build.settings)
         elif build.returncode == os.EX_OK:
             self.curval += 1
-            merge = PackageMerge(merge=build, scheduler=self._sched_iface)
+            merge = PackageMerge(
+                is_system_pkg=(build.pkg in self._deep_system_deps),
+                merge=build,
+                scheduler=self._sched_iface,
+            )
             self._running_tasks[id(merge)] = merge
             # By default, merge-wait only allows merge when no builds are executing.
             # As a special exception, dependencies on system packages are frequently
             # unspecified and will therefore force merge-wait.
-            is_system_pkg = build.pkg in self._deep_system_deps
             if not build.build_opts.buildpkgonly and (
-                "merge-wait" in build.settings.features or is_system_pkg
+                "merge-wait" in build.settings.features or merge.is_system_pkg
             ):
                 self._merge_wait_queue.append(merge)
-                if is_system_pkg:
+                if merge.is_system_pkg:
                     merge.addStartListener(self._system_merge_started)
             else:
                 self._task_queues.merge.add(merge)
@@ -1804,13 +1807,32 @@ def _schedule_tasks(self):
                 and not self._jobs
                 and not self._task_queues.merge
             ):
-                task = self._merge_wait_queue.popleft()
-                task.scheduler = self._sched_iface
-                self._merge_wait_scheduled.append(task)
-                self._task_queues.merge.add(task)
-                task.addExitListener(self._merge_wait_exit_handler)
-                self._status_display.merges = len(self._task_queues.merge)
-                state_change += 1
+                while self._merge_wait_queue:
+                    # If we added non-system packages to the merge queue in a
+                    # previous iteration of this loop, then for system packages we
+                    # need to come back later when the merge queue is empty.
+                    # TODO: Maybe promote non-system packages to the front of the
+                    # queue and process them within the current loop, though that
+                    # causes merge order to differ from the order builds finish.
+                    if (
+                        self._task_queues.merge
+                        and self._merge_wait_queue[0].is_system_pkg
+                    ):
+                        break
+                    task = self._merge_wait_queue.popleft()
+                    task.scheduler = self._sched_iface
+                    self._merge_wait_scheduled.append(task)
+                    self._task_queues.merge.add(task)
+                    task.addExitListener(self._merge_wait_exit_handler)
+                    self._status_display.merges = len(self._task_queues.merge)
+                    state_change += 1
+                    # For system packages, always serialize install regardless of
+                    # parallel-install, in order to mitigate failures triggered
+                    # by fragile states as in bug 256616. For other packages,
+                    # continue to populate self._task_queues.merge, which will
+                    # serialize install unless parallel-install is enabled.
+                    if task.is_system_pkg:
+                        break
 
             if self._schedule_tasks_imp():
                 state_change += 1

man/make.conf.5

@@ -652,9 +652,12 @@ terminal to view parallel-fetch progress.
 .TP
 .B parallel\-install
 Use finer\-grained locks when installing packages, allowing for greater
-parallelism. Note that \fIparallel\-install\fR currently has no effect
-unless \fImerge\-wait\fR is disabled. For additional parallelism,
-disable \fIebuild\-locks\fR.
+parallelism. For additional parallelism disable \fIebuild\-locks\fR.
+Also disable \fImerge\-wait\fR for additional parallelism if desired,
+but that increases the possibility of random build failures. When
+\fIparallel\-install\fR is used together with \fImerge\-wait\fR,
+parallel installation occurs in batches when there are no builds
+running.
 .TP
 .B pid\-sandbox
 Isolate the process space for the ebuild processes. This makes it