diff --git a/docs/changelog/2612.bugfix.rst b/docs/changelog/2612.bugfix.rst new file mode 100644 index 0000000000..c7b64462f0 --- /dev/null +++ b/docs/changelog/2612.bugfix.rst @@ -0,0 +1 @@ +Create session views of the build wheel/sdist into the :ref:`temp_dir` folder - by :user:`gaborbernat`. diff --git a/docs/config.rst b/docs/config.rst index da3812c522..df8155dfd1 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -172,7 +172,7 @@ Core .. conf:: :keys: temp_dir - :default: {tox_root}/.tmp + :default: {tox_root}/.temp Directory where to put tox temporary files. For example: we create a hard link (if possible, otherwise new copy) in this directory for the project package. This ensures tox works correctly when having parallel runs (as each session diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py index c87701f8c4..0315369a7e 100644 --- a/src/tox/config/sets.py +++ b/src/tox/config/sets.py @@ -199,8 +199,8 @@ def work_dir_builder(conf: Config, env_name: str | None) -> Path: # noqa: U100 self.add_config( keys=["temp_dir"], of_type=Path, - default=lambda conf, _: cast(Path, self["tox_root"]) / ".tmp", # noqa: U100, U101 - desc="temporary directory cleaned at start", + default=lambda conf, _: cast(Path, self["work_dir"]) / ".tmp", # noqa: U100, U101 + desc="a folder for temporary files (is not cleaned at start)", ) def _on_duplicate_conf(self, key: str, definition: ConfigDefinition[V]) -> None: # noqa: U100 diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py index b9255c1ce4..5e2bc1e410 100644 --- a/src/tox/tox_env/python/virtual_env/package/pyproject.py +++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py @@ -30,6 +30,7 @@ ) from tox.tox_env.register import ToxEnvRegister from tox.tox_env.runner import RunToxEnv +from tox.util.file_view import create_session_view from ..api import VirtualEnv from .util import dependencies_with_extras @@ -99,6 +100,7 @@ def __init__(self, create_args: ToxEnvCreateArgs) -> None: self._package_name: str | None = None self._pkg_lock = RLock() # can build only one package at a time self.root = self.conf["package_root"] + self._package_paths: set[Path] = set() @staticmethod def id() -> str: @@ -164,6 +166,9 @@ def _teardown(self) -> None: pass finally: executor.close() + for path in self._package_paths: + logging.debug("delete package %s", path) + path.unlink(missing_ok=True) super()._teardown() def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]: @@ -189,7 +194,10 @@ def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]: elif of_type == "sdist": self.setup() with self._pkg_lock: - package = SdistPackage(self._frontend.build_sdist(sdist_directory=self.pkg_dir).sdist, deps) + sdist = self._frontend.build_sdist(sdist_directory=self.pkg_dir).sdist + sdist = create_session_view(sdist, self._package_temp_path) + self._package_paths.add(sdist) + package = SdistPackage(sdist, deps) elif of_type in {"wheel", "editable"}: w_env = self._wheel_build_envs.get(for_env["wheel_build_env"]) if w_env is not None and w_env is not self: @@ -199,16 +207,22 @@ def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]: self.setup() method = "build_editable" if of_type == "editable" else "build_wheel" with self._pkg_lock: - path = getattr(self._frontend, method)( + wheel = getattr(self._frontend, method)( wheel_directory=self.pkg_dir, metadata_directory=self.meta_folder, config_settings=self._wheel_config_settings, ).wheel - package = (EditablePackage if of_type == "editable" else WheelPackage)(path, deps) + wheel = create_session_view(wheel, self._package_temp_path) + self._package_paths.add(wheel) + package = (EditablePackage if of_type == "editable" else WheelPackage)(wheel, deps) else: # pragma: no cover # for when we introduce new packaging types and don't implement raise TypeError(f"cannot handle package type {of_type}") # pragma: no cover return [package] + @property + def _package_temp_path(self) -> Path: + return self.core["temp_dir"] / "package" + def _load_deps(self, for_env: EnvConfigSet) -> list[Requirement]: # first check if this is statically available via PEP-621 deps = self._load_deps_from_static(for_env) diff --git a/src/tox/util/file_view.py b/src/tox/util/file_view.py new file mode 100644 index 0000000000..4b0e5e07b9 --- /dev/null +++ b/src/tox/util/file_view.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import logging +import os +import shutil +from itertools import chain +from os.path import commonpath +from pathlib import Path + + +def create_session_view(package: Path, temp_path: Path) -> Path: + """Allows using the file after you no longer holding a lock to it by moving it into a temp folder""" + # we'll number the active instances, and use the max value as session folder for a new build + # note we cannot change package names as PEP-491 (wheel binary format) + # is strict about file name structure + + temp_path.mkdir(parents=True, exist_ok=True) + exists = [i.name for i in temp_path.iterdir()] + file_id = max(chain((0,), (int(i) for i in exists if str(i).isnumeric()))) + session_dir = temp_path / str(file_id + 1) + session_dir.mkdir() + session_package = session_dir / package.name + + links = False # if we can do hard links do that, otherwise just copy + if hasattr(os, "link"): + try: + session_package.hardlink_to(package) + links = True + except (OSError, NotImplementedError): + pass + if not links: + shutil.copyfile(package, session_package) + operation = "links" if links else "copied" + common = commonpath((session_package, package)) + rel_session, rel_package = session_package.relative_to(common), package.relative_to(common) + logging.debug("package %s %s to %s (%s)", rel_session, operation, rel_package, common) + return session_package diff --git a/tests/tox_env/python/test_python_runner.py b/tests/tox_env/python/test_python_runner.py index 8fdeb4611b..797ad638d6 100644 --- a/tests/tox_env/python/test_python_runner.py +++ b/tests/tox_env/python/test_python_runner.py @@ -88,3 +88,14 @@ def test_journal_package_dir(tmp_path: Path) -> None: "type": "dir", }, } + + +def test_package_temp_dir_view(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: + project = tox_project({"tox.ini": "[testenv]\npackage=wheel"}) + result = project.run("r", "-vv", "-e", "py", "--root", str(demo_pkg_inline)) + result.assert_success() + wheel_name = "demo_pkg_inline-1.0.0-py3-none-any.whl" + session_path = Path(".tmp") / "package" / "1" / wheel_name + msg = f" D package {session_path} links to {Path('.pkg') / 'dist'/ wheel_name} ({project.path/ '.tox'}) " + assert msg in result.out + assert f" D delete package {project.path / '.tox' / session_path}" in result.out diff --git a/whitelist.txt b/whitelist.txt index 8f6089419c..794c630bb5 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -1,10 +1,37 @@ -0rc1 0x3 10ms 1s 2s 5s -a0 +BINPRM +BUF +BUFSIZE +COV +CTRL +Cfg +DEVNULL +E3 +E4 +EBADF +EIO +Hookimpl +Hookspec +IEXEC +INET +LIGHTRED +NOTSET +POSIX +Pep517 +Replacer +Repo +SIGINT +SIGKILL +SIGTERM +Sdist +TOX +Toml +Tox +VCS abi addinivalue addnodes @@ -15,9 +42,6 @@ autoclass autodoc autosectionlabel autouse -binprm -buf -bufsize cachetools canonicalize capfd @@ -30,32 +54,25 @@ chdir codec colorama commenters +commonpath conftest contnode copytree -cov cpus creq -ctrl cygwin deinit delenv -dev1 devenv devnull devpi -devrelease distinfo distlib divmod doc2path docname docutils -e3 -e4 -ebadf editables -eio entrypoints envs epilog @@ -81,22 +98,20 @@ getresult getsockname getsourcelines groupby +hardlink hookimpl -hookspec hookspecs -iexec -inet insort instream intersphinx isalpha isatty +isnumeric isspace iterdir levelname levelno libs -lightred linkcheck list2cmdline lvl @@ -106,7 +121,6 @@ namelist nitpicky nok nonlocal -notset nox objtype ov @@ -117,7 +131,6 @@ pluggy pos posargs posix -prerelease prereleases prj psutil @@ -148,10 +161,7 @@ rst sdist setenv shlex -sigint -sigkill signum -sigterm skipif splitter statemachine @@ -174,7 +184,6 @@ typeshed unescaped usedevelop usefixtures -vcs virtualenv whl win32