Skip to content

Commit

Permalink
Scheduler: Support parallel-install with merge-wait
Browse files Browse the repository at this point in the history
For system packages, always serialize install regardless of
parallel-install, in order to mitigate failures triggered
by fragile states as in bug 256616. For other packages,
continue to populate self._task_queues.merge, which will
serialize install unless parallel-install is enabled.

Fixes: 825db01 ("Add merge-wait FEATURES setting enabled by default")
Bug: https://bugs.gentoo.org/256616
Bug: https://bugs.gentoo.org/925213
Signed-off-by: Zac Medico <zmedico@gentoo.org>
  • Loading branch information
zmedico committed Feb 26, 2024
1 parent 50b1f27 commit a5762e8
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 16 deletions.
4 changes: 2 additions & 2 deletions lib/_emerge/PackageMerge.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 1999-2014 Gentoo Foundation
# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

from _emerge.CompositeTask import CompositeTask
Expand All @@ -7,7 +7,7 @@


class PackageMerge(CompositeTask):
__slots__ = ("merge", "postinst_failure")
__slots__ = ("is_system_pkg", "merge", "postinst_failure")

def _should_show_status(self):
return (
Expand Down
41 changes: 30 additions & 11 deletions lib/_emerge/Scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1519,17 +1519,20 @@ def _build_exit(self, build):
self._deallocate_config(build.settings)
elif build.returncode == os.EX_OK:
self.curval += 1
merge = PackageMerge(merge=build, scheduler=self._sched_iface)
merge = PackageMerge(
is_system_pkg=(build.pkg in self._deep_system_deps),
merge=build,
scheduler=self._sched_iface,
)
self._running_tasks[id(merge)] = merge
# By default, merge-wait only allows merge when no builds are executing.
# As a special exception, dependencies on system packages are frequently
# unspecified and will therefore force merge-wait.
is_system_pkg = build.pkg in self._deep_system_deps
if not build.build_opts.buildpkgonly and (
"merge-wait" in build.settings.features or is_system_pkg
"merge-wait" in build.settings.features or merge.is_system_pkg
):
self._merge_wait_queue.append(merge)
if is_system_pkg:
if merge.is_system_pkg:
merge.addStartListener(self._system_merge_started)
else:
self._task_queues.merge.add(merge)
Expand Down Expand Up @@ -1804,13 +1807,29 @@ def _schedule_tasks(self):
and not self._jobs
and not self._task_queues.merge
):
task = self._merge_wait_queue.popleft()
task.scheduler = self._sched_iface
self._merge_wait_scheduled.append(task)
self._task_queues.merge.add(task)
task.addExitListener(self._merge_wait_exit_handler)
self._status_display.merges = len(self._task_queues.merge)
state_change += 1
while self._merge_wait_queue:
# If we added non-system packages to the merge queue in a
# previous iteration of this loop, then for system packages
# we need to come back later when the merge queue is empty.
if (
self._task_queues.merge
and self._merge_wait_queue[0].is_system_pkg
):
break
task = self._merge_wait_queue.popleft()
task.scheduler = self._sched_iface
self._merge_wait_scheduled.append(task)
self._task_queues.merge.add(task)
task.addExitListener(self._merge_wait_exit_handler)
self._status_display.merges = len(self._task_queues.merge)
state_change += 1
# For system packages, always serialize install regardless of
# parallel-install, in order to mitigate failures triggered
# by fragile states as in bug 256616. For other packages,
# continue to populate self._task_queues.merge, which will
# serialize install unless parallel-install is enabled.
if task.is_system_pkg:
break

if self._schedule_tasks_imp():
state_change += 1
Expand Down
7 changes: 4 additions & 3 deletions man/make.conf.5
Original file line number Diff line number Diff line change
Expand Up @@ -652,9 +652,10 @@ terminal to view parallel-fetch progress.
.TP
.B parallel\-install
Use finer\-grained locks when installing packages, allowing for greater
parallelism. Note that \fIparallel\-install\fR currently has no effect
unless \fImerge\-wait\fR is disabled. For additional parallelism,
disable \fIebuild\-locks\fR.
parallelism. For additional parallelism disable \fIebuild\-locks\fR.
Also disable \fImerge\-wait\fR for additional parallelism if desired,
but this is not necessary because \fIparallel\-install\fR cooperates
with \fImerge\-wait\fR.
.TP
.B pid\-sandbox
Isolate the process space for the ebuild processes. This makes it
Expand Down

0 comments on commit a5762e8

Please sign in to comment.