Add BUILD_ID to metadata during binary packaging

Also create placeholder for new multi-instance package before actually
packaging to avoid race.

Signed-off-by: Sheng Yu <syu.os@protonmail.com>
Closes: #893
Signed-off-by: Michał Górny <mgorny@gentoo.org>

bin/gpkg-helper.py

@@ -19,7 +19,7 @@ def command_compose(args):
         sys.stderr.write("4 arguments are required, got %s\n" % len(args))
         return 1
 
-    cpv, binpkg_path, metadata_dir, image_dir = args
+    basename, binpkg_path, metadata_dir, image_dir = args
 
     if not os.path.isdir(metadata_dir):
         sys.stderr.write(usage)
@@ -31,7 +31,7 @@ def command_compose(args):
         sys.stderr.write("Argument 4 is not a directory: '%s'\n" % image_dir)
         return 1
 
-    gpkg_file = portage.gpkg.gpkg(portage.settings, cpv, binpkg_path)
+    gpkg_file = portage.gpkg.gpkg(portage.settings, basename, binpkg_path)
     metadata = gpkg_file._generate_metadata_from_dir(metadata_dir)
     gpkg_file.compress(image_dir, metadata)
     return os.EX_OK

bin/misc-functions.sh

@@ -512,6 +512,10 @@ __dyn_package() {
 		die "PORTAGE_BINPKG_TMPFILE is unset"
 	mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed"
 
+	if [[ ! -z "${BUILD_ID}" ]]; then
+		echo -n "${BUILD_ID}" > "${PORTAGE_BUILDDIR}"/build-info/BUILD_ID
+	fi
+
 	if [[ "${BINPKG_FORMAT}" == "xpak" ]]; then
 		local tar_options=""
 
@@ -547,7 +551,7 @@ __dyn_package() {
 	elif [[ "${BINPKG_FORMAT}" == "gpkg" ]]; then
 		PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
 			"${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}"/gpkg-helper.py compress \
-			"${CATEGORY}/${PF}" "${PORTAGE_BINPKG_TMPFILE}" "${PORTAGE_BUILDDIR}/build-info" "${D}"
+			"${PF}${BUILD_ID:+-${BUILD_ID}}" "${PORTAGE_BINPKG_TMPFILE}" "${PORTAGE_BUILDDIR}/build-info" "${D}"
 		if [[ $? -ne 0 ]]; then
 			rm -f "${PORTAGE_BINPKG_TMPFILE}"
 			die "Failed to create binpkg file"

bin/quickpkg

@@ -206,7 +206,7 @@ def quickpkg_atom(options, infos, arg, eout):
         finally:
             if have_lock:
                 dblnk.unlockdb()
-        pkg_info = bintree.inject(cpv, filename=binpkg_tmpfile)
+        pkg_info = bintree.inject(cpv, current_pkg_path=binpkg_tmpfile)
         # The pkg_info value ensures that the following getname call
         # returns the correct path when FEATURES=binpkg-multi-instance
         # is enabled, but fallback to cpv in case the inject call

lib/_emerge/Binpkg.py

@@ -246,7 +246,7 @@ def _verifier_exit(self, verifier):
 
         if self._fetched_pkg:
             pkg_path = self._bintree.getname(
-                self._bintree.inject(pkg.cpv, filename=self._fetched_pkg),
+                self._bintree.inject(pkg.cpv, current_pkg_path=self._fetched_pkg),
                 allocate_new=False,
             )
         else:

lib/_emerge/BinpkgFetcher.py

@@ -12,34 +12,39 @@
 import portage
 from portage import os
 from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
+from portage.const import SUPPORTED_XPAK_EXTENSIONS, SUPPORTED_GPKG_EXTENSIONS
 from portage.exception import FileNotFound, InvalidBinaryPackageFormat
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
 from portage.util._pty import _create_pty_or_pipe
 
 
 class BinpkgFetcher(CompositeTask):
 
-    __slots__ = ("pkg", "pretend", "logfile", "pkg_path")
+    __slots__ = ("pkg", "pretend", "logfile", "pkg_path", "pkg_allocated_path")
 
     def __init__(self, **kwargs):
         CompositeTask.__init__(self, **kwargs)
 
         pkg = self.pkg
         bintree = pkg.root_config.trees["bintree"]
-        binpkg_path = None
+        instance_key = bintree.dbapi._instance_key(pkg.cpv)
+        binpkg_format = bintree._remotepkgs[instance_key].get("BINPKG_FORMAT", None)
 
-        if bintree._remote_has_index:
-            instance_key = bintree.dbapi._instance_key(pkg.cpv)
+        if binpkg_format is None:
             binpkg_path = bintree._remotepkgs[instance_key].get("PATH")
-            if binpkg_path:
-                self.pkg_path = binpkg_path + ".partial"
+            if binpkg_path.endswith(SUPPORTED_XPAK_EXTENSIONS):
+                binpkg_format = "xpak"
+            elif binpkg_path.endswith(SUPPORTED_GPKG_EXTENSIONS):
+                binpkg_format = "gpkg"
             else:
-                self.pkg_path = (
-                    pkg.root_config.trees["bintree"].getname(pkg.cpv, allocate_new=True)
-                    + ".partial"
+                raise InvalidBinaryPackageFormat(
+                    f"Unsupported binary package format from '{binpkg_path}'"
                 )
-        else:
-            raise FileNotFound("Binary packages index not found")
+
+        self.pkg_allocated_path = pkg.root_config.trees["bintree"].getname(
+            pkg.cpv, allocate_new=True, remote_binpkg_format=binpkg_format
+        )
+        self.pkg_path = self.pkg_allocated_path + ".partial"
 
     def _start(self):
         fetcher = _BinpkgFetcherProcess(

lib/_emerge/BinpkgPrefetcher.py

@@ -11,6 +11,7 @@ class BinpkgPrefetcher(CompositeTask):
 
     __slots__ = ("pkg",) + (
         "pkg_path",
+        "pkg_allocated_path",
         "_bintree",
     )
 
@@ -23,6 +24,7 @@ def _start(self):
             scheduler=self.scheduler,
         )
         self.pkg_path = fetcher.pkg_path
+        self.pkg_allocated_path = fetcher.pkg_allocated_path
         self._start_task(fetcher, self._fetcher_exit)
 
     def _fetcher_exit(self, fetcher):
@@ -45,7 +47,11 @@ def _verifier_exit(self, verifier):
             self.wait()
             return
 
-        self._bintree.inject(self.pkg.cpv, filename=self.pkg_path)
+        self._bintree.inject(
+            self.pkg.cpv,
+            current_pkg_path=self.pkg_path,
+            allocated_pkg_path=self.pkg_allocated_path,
+        )
 
         self._current_task = None
         self.returncode = os.EX_OK

lib/_emerge/EbuildBinpkg.py

@@ -6,39 +6,34 @@
 
 import portage
 from portage import os
-from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
-from portage.exception import InvalidBinaryPackageFormat
 
 
 class EbuildBinpkg(CompositeTask):
     """
     This assumes that src_install() has successfully completed.
     """
 
-    __slots__ = ("pkg", "settings") + ("_binpkg_tmpfile", "_binpkg_info")
+    __slots__ = ("pkg", "settings") + (
+        "_binpkg_tmpfile",
+        "_binpkg_info",
+        "pkg_allocated_path",
+    )
 
     def _start(self):
         pkg = self.pkg
         root_config = pkg.root_config
         bintree = root_config.trees["bintree"]
-        binpkg_format = self.settings.get(
-            "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+        pkg_allocated_path, build_id = bintree.getname_build_id(
+            pkg.cpv, allocate_new=True
         )
-        if binpkg_format == "xpak":
-            binpkg_tmpfile = os.path.join(
-                bintree.pkgdir, pkg.cpv + ".tbz2." + str(portage.getpid())
-            )
-        elif binpkg_format == "gpkg":
-            binpkg_tmpfile = os.path.join(
-                bintree.pkgdir, pkg.cpv + ".gpkg.tar." + str(portage.getpid())
-            )
-        else:
-            raise InvalidBinaryPackageFormat(binpkg_format)
-        bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
 
-        self._binpkg_tmpfile = binpkg_tmpfile
+        self.pkg_allocated_path = pkg_allocated_path
+        self._binpkg_tmpfile = self.pkg_allocated_path + "." + str(portage.getpid())
         self.settings["PORTAGE_BINPKG_TMPFILE"] = self._binpkg_tmpfile
 
+        if "binpkg-multi-instance" in self.settings.features:
+            self.settings["BUILD_ID"] = str(build_id)
+
         package_phase = EbuildPhase(
             background=self.background,
             phase="package",
@@ -61,7 +56,11 @@ def _package_phase_exit(self, package_phase):
 
         pkg = self.pkg
         bintree = pkg.root_config.trees["bintree"]
-        self._binpkg_info = bintree.inject(pkg.cpv, filename=self._binpkg_tmpfile)
+        self._binpkg_info = bintree.inject(
+            pkg.cpv,
+            current_pkg_path=self._binpkg_tmpfile,
+            allocated_pkg_path=self.pkg_allocated_path,
+        )
 
         self._current_task = None
         self.returncode = os.EX_OK

lib/_emerge/Scheduler.py

@@ -975,7 +975,7 @@ async def _run_pkg_pretend(self, loop=None):
                         continue
 
                     if fetched:
-                        bintree.inject(x.cpv, filename=fetched)
+                        bintree.inject(x.cpv, current_pkg_path=fetched)
 
                     infloc = os.path.join(build_dir_path, "build-info")
                     ensure_dirs(infloc)