bintree: support file scheme for binhost src-uri

lib/_emerge/BinpkgFetcher.py

@@ -1,8 +1,6 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-import functools
-
 from _emerge.AsynchronousLock import AsynchronousLock
 from _emerge.CompositeTask import CompositeTask
 from _emerge.SpawnProcess import SpawnProcess
@@ -14,6 +12,7 @@
 from portage.binpkg import get_binpkg_format
 from portage.exception import FileNotFound
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.FileCopier import FileCopier
 from portage.util._pty import _create_pty_or_pipe
 
 
@@ -40,6 +39,22 @@ def __init__(self, **kwargs):
         self.pkg_path = self.pkg_allocated_path + ".partial"
 
     def _start(self):
+        self._start_task(
+            AsyncTaskFuture(future=self._main(), scheduler=self.scheduler),
+            self._main_exit,
+        )
+
+    async def _main(self) -> int:
+        """
+        Main coroutine which saves the binary package to self.pkg_path
+        and returns the exit status of the fetcher or copier.
+
+        @rtype: int
+        @return: Exit status of fetcher or copier.
+        """
+        pkg = self.pkg
+        bintree = pkg.root_config.trees["bintree"]
+
         fetcher = _BinpkgFetcherProcess(
             background=self.background,
             logfile=self.logfile,
@@ -52,47 +67,67 @@ def _start(self):
         if not self.pretend:
             portage.util.ensure_dirs(os.path.dirname(self.pkg_path))
             if "distlocks" in self.pkg.root_config.settings.features:
-                self._start_task(
-                    AsyncTaskFuture(future=fetcher.async_lock()),
-                    functools.partial(self._start_locked, fetcher),
-                )
-                return
-
-        self._start_task(fetcher, self._fetcher_exit)
-
-    def _start_locked(self, fetcher, lock_task):
-        self._assert_current(lock_task)
-        if lock_task.cancelled:
-            self._default_final_exit(lock_task)
-            return
-
-        lock_task.future.result()
-        self._start_task(fetcher, self._fetcher_exit)
-
-    def _fetcher_exit(self, fetcher):
-        self._assert_current(fetcher)
-        if not self.pretend and fetcher.returncode == os.EX_OK:
-            fetcher.sync_timestamp()
-        if fetcher.locked:
-            self._start_task(
-                AsyncTaskFuture(future=fetcher.async_unlock()),
-                functools.partial(self._fetcher_exit_unlocked, fetcher),
-            )
-        else:
-            self._fetcher_exit_unlocked(fetcher)
+                await fetcher.async_lock()
+
+        try:
+            if bintree._remote_has_index:
+                remote_metadata = bintree._remotepkgs[
+                    bintree.dbapi._instance_key(pkg.cpv)
+                ]
+                rel_uri = remote_metadata.get("PATH")
+                if not rel_uri:
+                    # Assume that the remote index is out of date. No path should
+                    # never happen in new portage versions.
+                    rel_uri = pkg.cpv + ".tbz2"
+                remote_base_uri = remote_metadata["BASE_URI"]
+                uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
+            else:
+                raise FileNotFound("Binary packages index not found")
 
-    def _fetcher_exit_unlocked(self, fetcher, unlock_task=None):
-        if unlock_task is not None:
-            self._assert_current(unlock_task)
-            if unlock_task.cancelled:
-                self._default_final_exit(unlock_task)
-                return
+            uri_parsed = urllib_parse_urlparse(uri)
 
-            unlock_task.future.result()
+            copier = None
+            if not self.pretend and uri_parsed.scheme in ("", "file"):
+                copier = FileCopier(
+                    src_path=uri_parsed.path,
+                    dest_path=self.pkg_path,
+                    scheduler=self.scheduler,
+                )
+                copier.start()
+                try:
+                    await copier.async_wait()
+                    copier.future.result()
+                except FileNotFoundError:
+                    await self.scheduler.async_output(
+                        f"!!! File not found: {uri_parsed.path}\n",
+                        log_file=self.logfile,
+                        background=self.background,
+                    )
+                finally:
+                    if copier.isAlive():
+                        copier.cancel()
 
-        self._current_task = None
-        self.returncode = fetcher.returncode
-        self._async_wait()
+            else:
+                fetcher.start()
+                try:
+                    await fetcher.async_wait()
+                finally:
+                    if fetcher.isAlive():
+                        fetcher.cancel()
+
+                if not self.pretend and fetcher.returncode == os.EX_OK:
+                    fetcher.sync_timestamp()
+        finally:
+            if fetcher.locked:
+                await fetcher.async_unlock()
+
+        return fetcher.returncode if copier is None else copier.returncode
+
+    def _main_exit(self, main_task):
+        if not main_task.cancelled:
+            # Use the fetcher or copier returncode.
+            main_task.returncode = main_task.future.result()
+        self._default_final_exit(main_task)
 
 
 class _BinpkgFetcherProcess(SpawnProcess):

lib/portage/dbapi/bintree.py

@@ -1411,15 +1411,18 @@ def _populate_remote(self, getbinpkg_refresh=True, pretend=False):
 
                 # Don't use urlopen for https, unless
                 # PEP 476 is supported (bug #469888).
-                if repo.fetchcommand is None and (
-                    parsed_url.scheme not in ("https",) or _have_pep_476()
-                ):
+                if (
+                    repo.fetchcommand is None or parsed_url.scheme in ("", "file")
+                ) and (parsed_url.scheme not in ("https",) or _have_pep_476()):
                     try:
-                        f = _urlopen(
-                            url, if_modified_since=local_timestamp, proxies=proxies
-                        )
-                        if hasattr(f, "headers") and f.headers.get("timestamp", ""):
-                            remote_timestamp = f.headers.get("timestamp")
+                        if parsed_url.scheme in ("", "file"):
+                            f = open(f"{parsed_url.path.rstrip('/')}/Packages", "rb")
+                        else:
+                            f = _urlopen(
+                                url, if_modified_since=local_timestamp, proxies=proxies
+                            )
+                            if hasattr(f, "headers") and f.headers.get("timestamp", ""):
+                                remote_timestamp = f.headers.get("timestamp")
                     except OSError as err:
                         if (
                             hasattr(err, "code") and err.code == 304

lib/portage/tests/emerge/conftest.py

@@ -814,21 +814,34 @@ def _rm_cachedir_and_pregen():
         )
 
         # Remove binrepos.conf and test PORTAGE_BINHOST.
-        def _replace_pkgdir_and_rm_binrepos_conf_file():
+        def _rm_pkgdir_and_rm_binrepos_conf_file():
             shutil.rmtree(pkgdir)
-            os.rename(binhost_dir, pkgdir)
             os.unlink(binrepos_conf_file)
 
         getbinpkgonly_fetchonly = Emerge(
             "-fe",
             "--getbinpkgonly",
             "dev-libs/A",
             env_mod={"PORTAGE_BINHOST": binhost_uri},
-            preparation=_replace_pkgdir_and_rm_binrepos_conf_file,
+            preparation=_rm_pkgdir_and_rm_binrepos_conf_file,
+        )
+
+        # Test bug 920537 binrepos.conf with local file src-uri.
+        def _rm_pkgdir_and_create_binrepos_conf_with_file_uri():
+            shutil.rmtree(pkgdir)
+            with open(binrepos_conf_file, "w") as f:
+                f.write("[test-binhost]\n")
+                f.write(f"sync-uri = file://{binhost_dir}\n")
+
+        getbinpkgonly_file_uri = Emerge(
+            "-fe",
+            "--getbinpkgonly",
+            "dev-libs/A",
+            preparation=_rm_pkgdir_and_create_binrepos_conf_with_file_uri,
         )
 
         fetch_sequence = PortageCommandSequence(
-            make_package, getbinpkgonly, getbinpkgonly_fetchonly
+            make_package, getbinpkgonly, getbinpkgonly_fetchonly, getbinpkgonly_file_uri
         )
         test_commands["binhost emerge"] = fetch_sequence
     yield test_commands

lib/portage/util/_async/AsyncTaskFuture.py

@@ -1,4 +1,4 @@
-# Copyright 2018-2021 Gentoo Foundation
+# Copyright 2018-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import os
@@ -20,6 +20,12 @@ def _start(self):
         self.future = asyncio.ensure_future(self.future, self.scheduler)
         self.future.add_done_callback(self._done_callback)
 
+    def isAlive(self):
+        """
+        Returns True if self.future is an asyncio.Future that is not done.
+        """
+        return isinstance(self.future, asyncio.Future) and not self.future.done()
+
     def _cancel(self):
         if not self.future.done():
             self.future.cancel()