From 525aca960d71915b908fb5918cdafc6d118e7dca Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Fri, 26 Aug 2022 11:38:16 +0200 Subject: [PATCH] feat(cli): change renku mv to respect datasets' datadir (#3071) --- .github/workflows/test_deploy.yml | 2 +- docs/cheatsheet/conf.py | 33 +++++++--- poetry.lock | 80 +++++++++++------------ renku/command/move.py | 49 +++++++++++++- renku/core/dataset/dataset.py | 17 +++-- renku/core/dataset/datasets_provenance.py | 16 ++++- tests/cli/test_move.py | 26 ++++---- 7 files changed, 149 insertions(+), 74 deletions(-) diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml index 5e413bb3fe..66b40c3098 100644 --- a/.github/workflows/test_deploy.yml +++ b/.github/workflows/test_deploy.yml @@ -179,7 +179,7 @@ jobs: - name: Set up Python 3.7 uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.9 - name: Install xetex run: | sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu jammy main restricted multiverse universe" diff --git a/docs/cheatsheet/conf.py b/docs/cheatsheet/conf.py index 341a5cee03..32a406ba6d 100644 --- a/docs/cheatsheet/conf.py +++ b/docs/cheatsheet/conf.py @@ -172,18 +172,31 @@ # sphinx type references only work for types that documentation is generated for # Suppress warnings for these types that are referenced but not documented themselves. nitpick_ignore = [ - ("py:class", "Path"), - ("py:class", "OID_TYPE"), - ("py:class", "optional"), - ("py:class", "persistent.Persistent"), - ("py:class", "DynamicProxy"), - ("py:class", "LocalClient"), - ("py:class", '"LocalClient"'), - ("py:class", "IClientDispatcher"), - ("py:class", "IDatasetGateway"), ("py:class", "CommandResult"), ("py:class", "CommunicationCallback"), + ("py:class", "datetime"), + ("py:class", "DiGraph"), + ("py:class", "DynamicProxy"), + ("py:class", "IActivityGateway"), + ("py:class", "IClientDispatcher"), ("py:class", "IDatabaseDispatcher"), - ("py:exc", "errors.ParameterError"), + ("py:class", "IDatasetGateway"), + ("py:class", "IPlanGateway"), + ("py:class", "LocalClient"), + ("py:class", "NoValueType"), + ("py:class", "OID_TYPE"), + ("py:class", "Path"), + ("py:class", "Persistent"), + ("py:class", "optional"), + ("py:class", '"LocalClient"'), ("py:class", '"ValueResolver"'), + ("py:exc", "errors.ParameterError"), +] + +nitpick_ignore_regex = [ + ("py:class", r"calamus.*"), + ("py:class", r"docker.*"), + ("py:class", r"marshmallow.*"), + ("py:class", r"persistent.*"), + ("py:class", r"yaml.*"), ] diff --git a/poetry.lock b/poetry.lock index 84249c3efe..cded48413a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -97,10 +97,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +tests_no_zope = ["cloudpickle", "pytest-mypy-plugins", "mypy", "six", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] +tests = ["cloudpickle", "zope.interface", "pytest-mypy-plugins", "mypy", "six", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] +docs = ["sphinx-notfound-page", "zope.interface", "sphinx", "furo"] +dev = ["cloudpickle", "pre-commit", "sphinx-notfound-page", "sphinx", "furo", "zope.interface", "pytest-mypy-plugins", "mypy", "six", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] [[package]] name = "babel" @@ -195,8 +195,8 @@ msgpack = ">=0.5.2" requests = "*" [package.extras] -filecache = ["lockfile (>=0.9)"] redis = ["redis (>=2.10.5)"] +filecache = ["lockfile (>=0.9)"] [[package]] name = "cachetools" @@ -777,9 +777,9 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +testing = ["importlib-resources (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "pytest-perf (>=0.9.2)", "flufl.flake8", "pyfakefs", "packaging", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] [[package]] name = "importlib-resources" @@ -903,10 +903,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" [package.extras] -cssselect = ["cssselect (>=0.7)"] -html5 = ["html5lib"] -htmlsoup = ["beautifulsoup4"] source = ["Cython (>=0.29.7)"] +htmlsoup = ["beautifulsoup4"] +html5 = ["html5lib"] +cssselect = ["cssselect (>=0.7)"] [[package]] name = "markupsafe" @@ -980,9 +980,9 @@ typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} typing-extensions = ">=3.10" [package.extras] -dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<2)"] reports = ["lxml"] +python2 = ["typed-ast (>=1.4.0,<2)"] +dmypy = ["psutil (>=4.0)"] [[package]] name = "mypy-extensions" @@ -1001,11 +1001,11 @@ optional = false python-versions = ">=3.7" [package.extras] -default = ["numpy (>=1.19)", "scipy (>=1.5,!=1.6.1)", "matplotlib (>=3.3)", "pandas (>=1.1)"] -developer = ["black (==21.5b1)", "pre-commit (>=2.12)"] -doc = ["sphinx (>=4.0,<5.0)", "pydata-sphinx-theme (>=0.6,<1.0)", "sphinx-gallery (>=0.9,<1.0)", "numpydoc (>=1.1)", "pillow (>=8.2)", "nb2plots (>=0.6)", "texext (>=0.6.6)"] -extra = ["lxml (>=4.5)", "pygraphviz (>=1.7)", "pydot (>=1.4.1)"] -test = ["pytest (>=6.2)", "pytest-cov (>=2.12)", "codecov (>=2.1)"] +test = ["codecov (>=2.1)", "pytest-cov (>=2.12)", "pytest (>=6.2)"] +extra = ["pydot (>=1.4.1)", "pygraphviz (>=1.7)", "lxml (>=4.5)"] +doc = ["texext (>=0.6.6)", "nb2plots (>=0.6)", "pillow (>=8.2)", "numpydoc (>=1.1)", "sphinx-gallery (>=0.9,<1.0)", "pydata-sphinx-theme (>=0.6,<1.0)", "sphinx (>=4.0,<5.0)"] +developer = ["pre-commit (>=2.12)", "black (==21.5b1)"] +default = ["pandas (>=1.1)", "matplotlib (>=3.3)", "scipy (>=1.5,!=1.6.1)", "numpy (>=1.19)"] [[package]] name = "numpy" @@ -1024,7 +1024,7 @@ optional = false python-versions = ">=3.7" [package.extras] -dev = ["pytest", "black", "mypy"] +dev = ["mypy", "black", "pytest"] [[package]] name = "owlrl" @@ -1108,8 +1108,8 @@ optional = true python-versions = ">=3.7" [package.extras] -docs = ["olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-issues (>=3.0.1)", "sphinx-removed-in", "sphinx-rtd-theme (>=1.0)", "sphinxext-opengraph"] -tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +tests = ["pytest-timeout", "pytest-cov", "pytest", "pyroma", "packaging", "olefile", "markdown2", "defusedxml", "coverage", "check-manifest"] +docs = ["sphinxext-opengraph", "sphinx-rtd-theme (>=1.0)", "sphinx-removed-in", "sphinx-issues (>=3.0.1)", "sphinx-copybutton", "sphinx (>=2.4)", "olefile"] [[package]] name = "plantweb" @@ -1133,8 +1133,8 @@ optional = true python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] -test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] +test = ["pytest (>=6)", "pytest-mock (>=3.6)", "pytest-cov (>=2.7)", "appdirs (==1.4.4)"] +docs = ["sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)", "proselint (>=0.10.2)", "furo (>=2021.7.5b38)"] [[package]] name = "pluggy" @@ -1242,7 +1242,7 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] -test = ["ipaddress", "mock", "enum34", "pywin32", "wmi"] +test = ["wmi", "pywin32", "enum34", "mock", "ipaddress"] [[package]] name = "ptyprocess" @@ -1396,7 +1396,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["railroad-diagrams", "jinja2"] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyreadline" @@ -1542,7 +1542,7 @@ python-versions = ">=3.7" pytest = ">=5.0" [package.extras] -dev = ["pre-commit", "tox", "pytest-asyncio"] +dev = ["pytest-asyncio", "tox", "pre-commit"] [[package]] name = "pytest-pep8" @@ -1595,9 +1595,9 @@ pytest = ">=6.2.0" pytest-forked = "*" [package.extras] -psutil = ["psutil (>=3.0)"] -setproctitle = ["setproctitle"] testing = ["filelock"] +setproctitle = ["setproctitle"] +psutil = ["psutil (>=3.0)"] [[package]] name = "python-dateutil" @@ -1712,8 +1712,8 @@ importlib-metadata = {version = ">=1.0", markers = "python_version < \"3.8\""} packaging = ">=20.4" [package.extras] +ocsp = ["requests (>=2.26.0)", "pyopenssl (==20.0.1)", "cryptography (>=36.0.1)"] hiredis = ["hiredis (>=1.0.0)"] -ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] [[package]] name = "renku-sphinx-theme" @@ -1748,8 +1748,8 @@ idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] [[package]] name = "requests-toolbelt" @@ -2019,8 +2019,8 @@ pygments = "*" sphinx = ">=2,<5" [package.extras] +testing = ["rinohtype", "bs4", "sphinx-testing", "pygments", "pytest-regressions", "pytest-cov", "pytest (>=3.6,<4)", "coverage"] code_style = ["pre-commit (==2.13.0)"] -testing = ["coverage", "pytest (>=3.6,<4)", "pytest-cov", "pytest-regressions", "pygments", "sphinx-testing", "bs4", "rinohtype"] [[package]] name = "sphinxcontrib-applehelp" @@ -2178,7 +2178,7 @@ python-versions = ">=3.7" [[package]] name = "tomlkit" -version = "0.11.2" +version = "0.11.3" description = "Style preserving TOML library" category = "main" optional = true @@ -2204,9 +2204,9 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] telegram = ["requests"] +notebook = ["ipywidgets (>=6)"] +dev = ["wheel", "twine", "py-make (>=0.1.0)"] [[package]] name = "transaction" @@ -2289,9 +2289,9 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] -brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +secure = ["ipaddress", "certifi", "idna (>=2.0.0)", "cryptography (>=1.3.4)", "pyOpenSSL (>=0.14)"] +brotli = ["brotlipy (>=0.6.0)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] [[package]] name = "vcrpy" @@ -2335,9 +2335,9 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] -optional = ["python-socks", "wsaccel"] test = ["websockets"] +optional = ["wsaccel", "python-socks"] +docs = ["sphinx-rtd-theme (>=0.5)", "Sphinx (>=3.4)"] [[package]] name = "werkzeug" @@ -2431,8 +2431,8 @@ optional = false python-versions = "*" [package.extras] +test = ["zope.testrunner", "zope.exceptions", "manuel", "docutils"] docs = ["sphinxcontrib-programoutput"] -test = ["docutils", "manuel", "zope.exceptions", "zope.testrunner"] [[package]] name = "zdaemon" @@ -3810,8 +3810,8 @@ tomli = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] tomlkit = [ - {file = "tomlkit-0.11.2-py3-none-any.whl", hash = "sha256:69e0675671a2eed1c08a53f342c955c4ead5d373a10f756219bf39f3d4f0018a"}, - {file = "tomlkit-0.11.2.tar.gz", hash = "sha256:d1b49c3e460f5910b22d799b13513504acb4f5fcaee01660ee66f07bd45a271c"}, + {file = "tomlkit-0.11.3-py3-none-any.whl", hash = "sha256:800628e7705ff7c7cc4395c29836c7073e55b9ec820e1fc696080f9c5591a789"}, + {file = "tomlkit-0.11.3.tar.gz", hash = "sha256:0ace4c975e0f3e6f71be8a2d61fe568777f1634bc80abff642cd3323ce709a0d"}, ] tornado = [ {file = "tornado-6.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72"}, diff --git a/renku/command/move.py b/renku/command/move.py index 7b25377e6f..9a72c0d171 100644 --- a/renku/command/move.py +++ b/renku/command/move.py @@ -26,7 +26,9 @@ from renku.core.dataset.dataset import move_files from renku.core.dataset.datasets_provenance import DatasetsProvenance from renku.core.interface.client_dispatcher import IClientDispatcher +from renku.core.interface.dataset_gateway import IDatasetGateway from renku.core.util import communication +from renku.core.util.os import get_relative_path, is_subpath def move_command(): @@ -48,12 +50,16 @@ def _move(sources, destination, force, verbose, to_dataset, client_dispatcher: I """ client = client_dispatcher.current_client - if to_dataset: - DatasetsProvenance().get_by_name(to_dataset, strict=True) - absolute_destination = _get_absolute_path(destination) absolute_sources = [_get_absolute_path(src) for src in sources] + if to_dataset: + target_dataset = DatasetsProvenance().get_by_name(to_dataset, strict=True) + if not is_subpath(absolute_destination, _get_absolute_path(target_dataset.get_datadir(client))): + raise errors.ParameterError( + f"Destination {destination} must be in {target_dataset.get_datadir(client)} when moving to a dataset." + ) + is_rename = len(absolute_sources) == 1 and ( not absolute_destination.exists() or (absolute_destination.is_file() and absolute_sources[0].is_file()) ) @@ -69,6 +75,7 @@ def _move(sources, destination, force, verbose, to_dataset, client_dispatcher: I raise errors.ParameterError("There are no files to move.") if not force: _check_existing_destinations(files.values()) + _warn_about_dataset_files(files) # NOTE: we don't check if source and destination are the same or if multiple sources are moved to the same # destination; git mv will check those and we raise if git mv fails. @@ -217,6 +224,42 @@ def _warn_about_git_filters(files, client_dispatcher: IClientDispatcher): ) +@inject.autoparams() +def _warn_about_dataset_files(files, dataset_gateway: IDatasetGateway, client_dispatcher: IClientDispatcher): + """Check if any of the files are part of a dataset. + + Args: + files: Files to check. + dataset_gateway(IDatasetGateway): Injected dataset gateway. + client_dispatcher(IClientDispatcher): Injected client dispatcher. + """ + client = client_dispatcher.current_client + + found = [] + for dataset in dataset_gateway.get_all_active_datasets(): + for src, dst in files.items(): + relative_src = get_relative_path(src, client.path) + if not relative_src: + continue + + found_file = dataset.find_file(relative_src) + if not found_file: + continue + if not found_file.is_external and not is_subpath(dst, client.path / dataset.get_datadir(client)): + found.append(str(src)) + + if not found: + return + + found_str = "\n\t".join(found) + communication.confirm( + msg="You are trying to move dataset files out of a datasets data directory. " + f"These files will be removed from the source dataset:\n\t{found_str}", + abort=True, + warning=True, + ) + + def _show_moved_files(client_path, files): for path in sorted(files): src = path.relative_to(client_path) diff --git a/renku/core/dataset/dataset.py b/renku/core/dataset/dataset.py index e27a339f15..ae0fbaadbe 100644 --- a/renku/core/dataset/dataset.py +++ b/renku/core/dataset/dataset.py @@ -42,7 +42,7 @@ from renku.core.util.dispatcher import get_client, get_database from renku.core.util.git import clone_repository, get_cache_directory_for_repository, get_git_user from renku.core.util.metadata import is_external_file, read_credentials, store_credentials -from renku.core.util.os import delete_file, get_safe_relative_path +from renku.core.util.os import delete_file, get_safe_relative_path, is_subpath from renku.core.util.tabulate import tabulate from renku.core.util.urls import get_slug from renku.core.util.util import NO_VALUE, NoValueType @@ -391,7 +391,7 @@ def export_dataset(name, provider_name, tag, client_dispatcher: IClientDispatche # TODO: all these callbacks are ugly, improve in #737 config_key_secret = "access_token" - dataset = datasets_provenance.get_by_name(name, strict=True, immutable=True) + dataset: Optional[Dataset] = datasets_provenance.get_by_name(name, strict=True, immutable=True) provider = ProviderFactory.from_name(provider_name) @@ -765,7 +765,7 @@ def show_dataset(name: str, tag: Optional[str] = None): dict: JSON dictionary of dataset details. """ datasets_provenance = DatasetsProvenance() - dataset = datasets_provenance.get_by_name(name, strict=True) + dataset: Optional[Dataset] = datasets_provenance.get_by_name(name, strict=True) if tag is None: return DatasetDetailsJson().dump(dataset) @@ -876,7 +876,7 @@ def move_files( for src, dst in files.items(): src = src.relative_to(client.path) dst = dst.relative_to(client.path) - # NOTE: Files are moved at this point, so, we use can use dst + # NOTE: Files are moved at this point, so, we can use dst new_dataset_file = DatasetFile.from_path(client, dst) for dataset in datasets: @@ -885,12 +885,17 @@ def move_files( modified_datasets[dataset.name] = dataset new_dataset_file.based_on = removed.based_on new_dataset_file.source = removed.source - if not to_dataset: + + if not to_dataset and ( + new_dataset_file.is_external + or is_subpath(client.path / dst, client.path / dataset.get_datadir(client)) + ): dataset.add_or_update_files(new_dataset_file) # NOTE: Update dataset if it contains a destination that is being overwritten modified = dataset.find_file(dst) - if modified: + added = is_subpath(client.path / dst, client.path / dataset.get_datadir(client)) + if modified or added: modified_datasets[dataset.name] = dataset dataset.add_or_update_files(new_dataset_file) diff --git a/renku/core/dataset/datasets_provenance.py b/renku/core/dataset/datasets_provenance.py index 7f4b0ec7a0..ac27281464 100644 --- a/renku/core/dataset/datasets_provenance.py +++ b/renku/core/dataset/datasets_provenance.py @@ -18,7 +18,7 @@ """Datasets Provenance.""" from datetime import datetime -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, List, Literal, Optional, Union, overload from uuid import UUID from renku.command.command_builder.command import inject @@ -57,7 +57,19 @@ def get_by_id(self, id: str, immutable: bool = False) -> Optional["Dataset"]: return dataset.copy() return None - def get_by_name(self, name: str, immutable: bool = False, strict: bool = False) -> Optional["Dataset"]: + @overload + def get_by_name( + self, name: str, *, immutable: bool = False, strict: Literal[False] = False + ) -> Optional["Dataset"]: # noqa: D102 + ... + + @overload + def get_by_name(self, name: str, *, immutable: bool = False, strict: Literal[True]) -> "Dataset": # noqa: D102 + ... + + def get_by_name( + self, name: str, immutable: bool = False, strict: bool = False + ) -> Union[Optional["Dataset"], "Dataset"]: """Return a dataset by its name.""" dataset = self.dataset_gateway.get_by_name(name) if not dataset: diff --git a/tests/cli/test_move.py b/tests/cli/test_move.py index f1d08cb300..2214b6bfad 100644 --- a/tests/cli/test_move.py +++ b/tests/cli/test_move.py @@ -130,10 +130,6 @@ def test_move_empty_source(runner, client): assert "Invalid parameter value - There are no files to move" in result.output -@pytest.mark.skip( - reason="Test moves dataset file outside of datadir, " - "see https://github.com/SwissDataScienceCenter/renku-python/issues/3061" -) def test_move_dataset_file(runner, client_with_datasets, directory_tree_files, load_dataset_with_injection): """Test move of a file that belongs to a dataset.""" for path in directory_tree_files: @@ -142,9 +138,12 @@ def test_move_dataset_file(runner, client_with_datasets, directory_tree_files, l dataset_before = load_dataset_with_injection("dataset-2", client_with_datasets) - assert 0 == runner.invoke(cli, ["mv", "data", "files"], catch_exceptions=False).exit_code + result = runner.invoke(cli, ["mv", "data", "files"], input="y", catch_exceptions=False) + assert 0 == result.exit_code, format_result_exception(result) + assert "Warning: You are trying to move dataset files out of a datasets data directory" in result.output - assert 0 == runner.invoke(cli, ["doctor"], catch_exceptions=False).exit_code + result = runner.invoke(cli, ["doctor"], catch_exceptions=False) + assert 0 == result.exit_code, format_result_exception(result) # Check immutability dataset_after = load_dataset_with_injection("dataset-2", client_with_datasets) @@ -158,9 +157,7 @@ def test_move_dataset_file(runner, client_with_datasets, directory_tree_files, l assert dst.exists() file = dataset_after.find_file(dst) - assert file - assert str(dst) in file.entity.id - assert not file.is_external + assert not file @pytest.mark.parametrize("args", [[], ["--to-dataset", "dataset-2"]]) @@ -193,6 +190,10 @@ def test_move_to_existing_destination_in_a_dataset(runner, client_with_datasets, client_with_datasets.repository.add(all=True) client_with_datasets.repository.commit("source file") + result = runner.invoke(cli, ["mv", "-f", "--to-dataset", "dataset-2", "source", "new_file"]) + assert 2 == result.exit_code, format_result_exception(result) + assert "Destination new_file must be in data/dataset-2 when moving to a dataset." in result.output + dst = os.path.join("data", "dataset-2", "file1") dataset_before = load_dataset_with_injection("dataset-2", client_with_datasets) @@ -234,7 +235,8 @@ def test_move_external_files( """Test move of external files (symlinks).""" assert 0 == runner.invoke(cli, ["dataset", "add", "-c", "--external", "my-dataset", str(directory_tree)]).exit_code - assert 0 == runner.invoke(cli, ["mv", os.path.join(DATA_DIR, "my-dataset"), destination]).exit_code + result = runner.invoke(cli, ["mv", os.path.join(DATA_DIR, "my-dataset"), destination]) + assert 0 == result.exit_code, format_result_exception(result) for path in directory_tree_files: dst = Path(destination) / directory_tree.name / path @@ -268,7 +270,7 @@ def test_move_between_datasets( source = Path("data") / "dataset-1" destination = Path("data") / "dataset-3" - result = runner.invoke(cli, ["mv", str(source), str(destination), "--to-dataset", "dataset-3"]) + result = runner.invoke(cli, ["mv", "-f", str(source), str(destination), "--to-dataset", "dataset-3"]) assert 0 == result.exit_code, format_result_exception(result) assert not source.exists() @@ -288,7 +290,7 @@ def test_move_between_datasets( src1 = os.path.join("data", "dataset-3", directory_tree.name, "dir1") dst1 = os.path.join("data", "dataset-1") shutil.rmtree(dst1, ignore_errors=True) # NOTE: Remove directory to force a rename - assert 0 == runner.invoke(cli, ["mv", src1, dst1, "--to-dataset", "dataset-1"]).exit_code + assert 0 == runner.invoke(cli, ["mv", "-f", src1, dst1, "--to-dataset", "dataset-1"]).exit_code src2 = os.path.join("data", "dataset-3", directory_tree.name, "file1") dst2 = os.path.join("data", "dataset-2") (client.path / dst2).mkdir(parents=True, exist_ok=True)