From 825db01b91a37dcd9890ee5bf9f462ea524ac5cc Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Mon, 15 Jan 2024 21:06:36 -0800 Subject: [PATCH] Add merge-wait FEATURES setting enabled by default Add a new merge-wait FEATURES setting, enabled by default, which prevents packages from being merged while other packages are building. Previously, this behavior was already implemented for packages that satisfy direct or indirect dependencies of the system set. This feature can be disabled only for other packages which do not satisfy direct or indirect dependencies of the system set, in order to trade the possibility of random build failures for greater parallelism. Currently, it is known that having merge-wait disabled can cause "random" build failures for builds using setuptools when setuptools plugins are merged during the build. Bug: https://bugs.gentoo.org/663324 Signed-off-by: Zac Medico --- cnf/make.globals | 4 ++-- lib/_emerge/Scheduler.py | 14 ++++++++------ lib/portage/const.py | 3 ++- man/make.conf.5 | 16 +++++++++++++--- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/cnf/make.globals b/cnf/make.globals index af2642988a..2bb7a65593 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -1,4 +1,4 @@ -# Copyright 1999-2020 Gentoo Authors +# Copyright 1999-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # System-wide defaults for the Portage system @@ -77,7 +77,7 @@ FETCHCOMMAND_SFTP="bash -c \"x=\\\${2#sftp://} ; host=\\\${x%%/*} ; port=\\\${ho FEATURES="assume-digests binpkg-docompress binpkg-dostrip binpkg-logs binpkg-multi-instance buildpkg-live config-protect-if-modified distlocks ebuild-locks - fixlafiles ipc-sandbox merge-sync multilib-strict + fixlafiles ipc-sandbox merge-sync merge-wait multilib-strict network-sandbox news parallel-fetch pkgdir-index-trusted pid-sandbox preserve-libs protect-owned qa-unresolved-soname-deps sandbox sfperms strict diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index ae01214d32..9950792dc9 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -1521,14 +1521,16 @@ def _build_exit(self, build): self.curval += 1 merge = PackageMerge(merge=build, scheduler=self._sched_iface) self._running_tasks[id(merge)] = merge - if ( - not build.build_opts.buildpkgonly - and build.pkg in self._deep_system_deps + # By default, merge-wait only allows merge when no builds are executing. + # As a special exception, dependencies on system packages are frequently + # unspecified and will therefore force merge-wait. + is_system_pkg = build.pkg in self._deep_system_deps + if not build.build_opts.buildpkgonly and ( + "merge-wait" in build.settings.features or is_system_pkg ): - # Since dependencies on system packages are frequently - # unspecified, merge them only when no builds are executing. self._merge_wait_queue.append(merge) - merge.addStartListener(self._system_merge_started) + if is_system_pkg: + merge.addStartListener(self._system_merge_started) else: self._task_queues.merge.add(merge) merge.addExitListener(self._merge_exit) diff --git a/lib/portage/const.py b/lib/portage/const.py index bf310bb6e0..1f468a1abf 100644 --- a/lib/portage/const.py +++ b/lib/portage/const.py @@ -1,5 +1,5 @@ # portage: Constants -# Copyright 1998-2023 Gentoo Authors +# Copyright 1998-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os @@ -194,6 +194,7 @@ "keepwork", "lmirror", "merge-sync", + "merge-wait", "metadata-transfer", "mirror", "mount-sandbox", diff --git a/man/make.conf.5 b/man/make.conf.5 index f527166da3..20dc9cb52d 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -1,4 +1,4 @@ -.TH "MAKE.CONF" "5" "Mar 2023" "Portage @VERSION@" "Portage" +.TH "MAKE.CONF" "5" "Jan 2024" "Portage @VERSION@" "Portage" .SH "NAME" make.conf \- custom settings for Portage .SH "SYNOPSIS" @@ -565,6 +565,15 @@ After a package is merged or unmerged, sync relevant files to disk in order to avoid data\-loss in the event of a power failure. This feature is enabled by default. .TP +.B merge\-wait +Wait for all builds to complete before merging new packages, which only +matters when using the \fBemerge\fR(1) \fB\-\-jobs\fR option. This +feature is always effectively enabled and it cannot be disabled for +packages that satisfy direct or indirect dependencies of the system +set. For other packages, it can be disabled in order to trade the +possibility of random build failures for greater parallelism. +This feature is enabled by default. +.TP .B metadata\-transfer Automatically perform a metadata transfer when `emerge \-\-sync` is run. In versions of portage >=2.1.5, this feature is disabled by @@ -637,8 +646,9 @@ terminal to view parallel-fetch progress. .TP .B parallel\-install Use finer\-grained locks when installing packages, allowing for greater -parallelization. For additional parallelization, disable -\fIebuild\-locks\fR. +parallelism. Note that \fIparallel\-install\fR currently has no effect +unless \fImerge\-wait\fR is disabled. For additional parallelism, +disable \fIebuild\-locks\fR. .TP .B pid\-sandbox Isolate the process space for the ebuild processes. This makes it