From f3aa2fe1b82559ea9f9de50255a7cb88422cf47c Mon Sep 17 00:00:00 2001 From: Nick Sorros Date: Fri, 25 Mar 2022 10:40:13 +0200 Subject: [PATCH 1/7] Raise informative missing s3 extra --- dvc/output.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dvc/output.py b/dvc/output.py index 13fd8e737a..73bd254404 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -27,6 +27,7 @@ from .fs.hdfs import HDFSFileSystem from .fs.local import LocalFileSystem from .fs.s3 import S3FileSystem +from .fs.base import RemoteMissingDepsError from .hash_info import HashInfo from .istextfile import istextfile from .objects.errors import ObjectFormatError @@ -871,6 +872,8 @@ def _collect_used_dir_cache( try: self.get_dir_cache(jobs=jobs, remote=remote) + except RemoteMissingDepsError as ex: + raise ex except DvcException: logger.debug(f"failed to pull cache for '{self}'") From 2a763416c815a00bf0e9bb5455bb7e5fe5f3aea8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 25 Mar 2022 08:41:31 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dvc/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/output.py b/dvc/output.py index 73bd254404..6fec061e63 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -24,10 +24,10 @@ from .data.transfer import transfer as otransfer from .data.tree import Tree from .fs import get_cloud_fs +from .fs.base import RemoteMissingDepsError from .fs.hdfs import HDFSFileSystem from .fs.local import LocalFileSystem from .fs.s3 import S3FileSystem -from .fs.base import RemoteMissingDepsError from .hash_info import HashInfo from .istextfile import istextfile from .objects.errors import ObjectFormatError From 622fe45e02a67b606d80e0be408761fc9d97784b Mon Sep 17 00:00:00 2001 From: Nick Sorros Date: Tue, 5 Apr 2022 20:23:18 +0300 Subject: [PATCH 3/7] Add test for remote missing when adding data dir --- tests/func/test_data_cloud.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index eea0e549b4..5665a7228e 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -1,6 +1,7 @@ import logging import os import shutil +from unittest.mock import patch import pytest from flaky.flaky_decorator import flaky @@ -9,6 +10,7 @@ from dvc.cli import main from dvc.data.db.local import LocalObjectDB from dvc.external_repo import clean_repos +from dvc.fs.base import RemoteMissingDepsError from dvc.objects.db import ObjectDB from dvc.stage.exceptions import StageNotFound from dvc.testing.test_remote import ( # noqa, pylint: disable=unused-import @@ -570,3 +572,19 @@ def test_target_remote(tmp_dir, dvc, make_remote): "6b18131dc289fd37006705affe961ef8.dir", "b8a9f715dbb64fd5c56e7783c6820a61", } + + +def test_remote_missing_data_dir(tmp_dir, scm, dvc, make_remote): + make_remote("default", default=True) + tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") + dvc.push() + + remove("dir") + remove(dvc.odb.local.cache_dir) + + with patch( + "dvc.fs.base.FileSystem._check_requires", + side_effect=RemoteMissingDepsError("remote missing"), + ): + with pytest.raises(RemoteMissingDepsError): + dvc.pull() From c9431241d1fe4e7ed021ee343cd86f4cac32be0f Mon Sep 17 00:00:00 2001 From: Nick Sorros Date: Wed, 6 Apr 2022 08:33:59 +0300 Subject: [PATCH 4/7] Reraise RemoteMissingDepsError as is MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Redzyński --- dvc/output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/output.py b/dvc/output.py index 6fec061e63..a532f98603 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -872,8 +872,8 @@ def _collect_used_dir_cache( try: self.get_dir_cache(jobs=jobs, remote=remote) - except RemoteMissingDepsError as ex: - raise ex + except RemoteMissingDepsError: + raise except DvcException: logger.debug(f"failed to pull cache for '{self}'") From 69138aa193be3040b8b41870e1b32700c6a5cf3a Mon Sep 17 00:00:00 2001 From: Nick Sorros Date: Wed, 6 Apr 2022 11:04:46 +0300 Subject: [PATCH 5/7] Patch get_remote_odb instead --- tests/func/test_data_cloud.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 5665a7228e..2bf53c7d38 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -575,15 +575,16 @@ def test_target_remote(tmp_dir, dvc, make_remote): def test_remote_missing_data_dir(tmp_dir, scm, dvc, make_remote): - make_remote("default", default=True) tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") - dvc.push() + with dvc.config.edit() as conf: + conf["remote"]["s3"] = {"url": "s3://bucket/name"} + conf["core"] = {"remote": "s3"} remove("dir") remove(dvc.odb.local.cache_dir) with patch( - "dvc.fs.base.FileSystem._check_requires", + "dvc.data_cloud.DataCloud.get_remote_odb", side_effect=RemoteMissingDepsError("remote missing"), ): with pytest.raises(RemoteMissingDepsError): From 8a37f70e36368f090552b10f0ba6acb874b1c8ad Mon Sep 17 00:00:00 2001 From: Nick Sorros Date: Thu, 7 Apr 2022 08:19:14 +0300 Subject: [PATCH 6/7] Move test to output and use mocker --- tests/func/test_data_cloud.py | 19 ------------------- tests/unit/output/test_output.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 2bf53c7d38..eea0e549b4 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -1,7 +1,6 @@ import logging import os import shutil -from unittest.mock import patch import pytest from flaky.flaky_decorator import flaky @@ -10,7 +9,6 @@ from dvc.cli import main from dvc.data.db.local import LocalObjectDB from dvc.external_repo import clean_repos -from dvc.fs.base import RemoteMissingDepsError from dvc.objects.db import ObjectDB from dvc.stage.exceptions import StageNotFound from dvc.testing.test_remote import ( # noqa, pylint: disable=unused-import @@ -572,20 +570,3 @@ def test_target_remote(tmp_dir, dvc, make_remote): "6b18131dc289fd37006705affe961ef8.dir", "b8a9f715dbb64fd5c56e7783c6820a61", } - - -def test_remote_missing_data_dir(tmp_dir, scm, dvc, make_remote): - tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") - with dvc.config.edit() as conf: - conf["remote"]["s3"] = {"url": "s3://bucket/name"} - conf["core"] = {"remote": "s3"} - - remove("dir") - remove(dvc.odb.local.cache_dir) - - with patch( - "dvc.data_cloud.DataCloud.get_remote_odb", - side_effect=RemoteMissingDepsError("remote missing"), - ): - with pytest.raises(RemoteMissingDepsError): - dvc.pull() diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py index 46e892b189..cf187d5b49 100644 --- a/tests/unit/output/test_output.py +++ b/tests/unit/output/test_output.py @@ -5,9 +5,11 @@ from funcy import first from voluptuous import MultipleInvalid, Schema +from dvc.fs.base import RemoteMissingDepsError from dvc.ignore import _no_match from dvc.output import CHECKSUM_SCHEMA, Output from dvc.stage import Stage +from dvc.utils.fs import remove def test_save_missing(dvc, mocker): @@ -100,3 +102,20 @@ def test_get_used_objs(exists, expected_message, mocker, caplog): with caplog.at_level(logging.WARNING, logger="dvc"): assert {} == output.get_used_objs() assert first(caplog.messages) == expected_message + + +def test_remote_missing_data_dir(tmp_dir, scm, dvc, mocker): + tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") + with dvc.config.edit() as conf: + conf["remote"]["s3"] = {"url": "s3://bucket/name"} + conf["core"] = {"remote": "s3"} + + remove("dir") + remove(dvc.odb.local.cache_dir) + + with mocker.patch( + "dvc.data_cloud.DataCloud.get_remote_odb", + side_effect=RemoteMissingDepsError("remote missing"), + ): + with pytest.raises(RemoteMissingDepsError): + dvc.pull() From 3f75112281e70dbb16ca7d483fd4c33b38fa5c59 Mon Sep 17 00:00:00 2001 From: Nick Sorros Date: Mon, 11 Apr 2022 16:10:53 +0300 Subject: [PATCH 7/7] Update test name to be more informative MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Redzyński --- tests/unit/output/test_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py index cf187d5b49..c539ee3106 100644 --- a/tests/unit/output/test_output.py +++ b/tests/unit/output/test_output.py @@ -104,7 +104,7 @@ def test_get_used_objs(exists, expected_message, mocker, caplog): assert first(caplog.messages) == expected_message -def test_remote_missing_data_dir(tmp_dir, scm, dvc, mocker): +def test_remote_missing_depenency_on_dir_pull(tmp_dir, scm, dvc, mocker): tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") with dvc.config.edit() as conf: conf["remote"]["s3"] = {"url": "s3://bucket/name"}