From 8cfdbed6f90146ae78536396dbbed6712bb94ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Thu, 16 Jan 2020 20:29:16 +0000 Subject: [PATCH 1/5] update: handle git-tracked imported files Fixes #2976 --- dvc/dependency/repo.py | 9 +++++++-- tests/func/test_update.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 3bcfd3e65d..66c8b278fc 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -142,5 +142,10 @@ def download(self, to): ) def update(self): - with self._make_repo(rev_lock=None) as repo: - self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev() + try: + with self._make_repo(rev_lock=None) as repo: + rev = repo.scm.get_rev() + except NotDvcRepoError: + clone_path = cached_clone(**self.def_repo) + rev = SCM(clone_path).get_rev() + self.def_repo[self.PARAM_REV_LOCK] = rev diff --git a/tests/func/test_update.py b/tests/func/test_update.py index 6f483fff0a..631c0fb055 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -172,3 +172,35 @@ def test_update_import_url(tmp_dir, dvc, tmp_path_factory): assert dst.is_file() assert dst.read_text() == "updated file content" + + +def test_update_git_tracked(tmp_dir, dvc, erepo_dir): + with erepo_dir.chdir(): + erepo_dir.scm.repo.index.remove([".dvc"], r=True) + shutil.rmtree(".dvc") + erepo_dir.scm_gen("file", "first version") + erepo_dir.scm.add(["file"]) + erepo_dir.scm.commit("first version") + + stage = dvc.imp(fspath(erepo_dir), "file", "file") + + # Just to make sure it doesn't crash + dvc.update(stage.path) + + assert (tmp_dir / "file").read_text() == "first version" + + with erepo_dir.chdir(): + erepo_dir.scm.repo.index.remove(["file"]) + os.remove("file") + erepo_dir.scm_gen("file", "second version") + erepo_dir.scm.add(["file"]) + erepo_dir.scm.commit("x") + + # Caching in external repos doesn't see upstream updates within single + # cli call, so we need to clean the caches to see the changes. + clean_repos() + + dvc.update(stage.path) + + assert (tmp_dir / "file").read_text() == "second version" + assert dvc.status([stage.path]) == {} From a174a60c7f8e40334fb9cff0b5895a419f5b1da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Fri, 17 Jan 2020 09:00:10 +0000 Subject: [PATCH 2/5] simplify test --- tests/func/test_update.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/func/test_update.py b/tests/func/test_update.py index 631c0fb055..a7489069e6 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -190,17 +190,12 @@ def test_update_git_tracked(tmp_dir, dvc, erepo_dir): assert (tmp_dir / "file").read_text() == "first version" with erepo_dir.chdir(): - erepo_dir.scm.repo.index.remove(["file"]) - os.remove("file") - erepo_dir.scm_gen("file", "second version") - erepo_dir.scm.add(["file"]) - erepo_dir.scm.commit("x") + erepo_dir.scm_gen("file", "second version", commit="update file") # Caching in external repos doesn't see upstream updates within single # cli call, so we need to clean the caches to see the changes. clean_repos() - dvc.update(stage.path) + dvc.update("file.dvc") assert (tmp_dir / "file").read_text() == "second version" - assert dvc.status([stage.path]) == {} From 73766984da2507fd0d6af55b40389363dbc0f2f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Fri, 17 Jan 2020 09:45:40 +0000 Subject: [PATCH 3/5] create git_repo fixture --- tests/dir_helpers.py | 27 ++++++++++++++++++++++++++- tests/func/test_update.py | 20 ++++++++------------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index c531e4b8cc..1e6d8ce308 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -54,7 +54,15 @@ from dvc.compat import fspath, fspath_py35 -__all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"] +__all__ = [ + "tmp_dir", + "scm", + "dvc", + "repo_template", + "run_copy", + "erepo_dir", + "git_dir", +] REPO_TEMPLATE = { "foo": "foo", "bar": "bar", @@ -285,3 +293,20 @@ def erepo_dir(tmp_path_factory, monkeypatch): path.dvc.close() return path + + +@pytest.fixture +def git_dir(tmp_path_factory, monkeypatch): + from dvc.scm.git import Git + + path = TmpDir(fspath_py35(tmp_path_factory.mktemp("git_dir"))) + + # Create git repo + with path.chdir(): + _git_init() + + try: + path.scm = Git(fspath(path)) + yield path + finally: + path.scm.close() diff --git a/tests/func/test_update.py b/tests/func/test_update.py index a7489069e6..6d8ae9da21 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -174,28 +174,24 @@ def test_update_import_url(tmp_dir, dvc, tmp_path_factory): assert dst.read_text() == "updated file content" -def test_update_git_tracked(tmp_dir, dvc, erepo_dir): - with erepo_dir.chdir(): - erepo_dir.scm.repo.index.remove([".dvc"], r=True) - shutil.rmtree(".dvc") - erepo_dir.scm_gen("file", "first version") - erepo_dir.scm.add(["file"]) - erepo_dir.scm.commit("first version") +def test_update_git_tracked(tmp_dir, dvc, git_dir): + with git_dir.chdir(): + git_dir.scm_gen("file", "first version", commit="create") - stage = dvc.imp(fspath(erepo_dir), "file", "file") + tmp_dir.dvc.imp(fspath(git_dir), "file", "file") # Just to make sure it doesn't crash - dvc.update(stage.path) + tmp_dir.dvc.update("file.dvc") assert (tmp_dir / "file").read_text() == "first version" - with erepo_dir.chdir(): - erepo_dir.scm_gen("file", "second version", commit="update file") + with git_dir.chdir(): + git_dir.scm_gen("file", "second version", commit="update file") # Caching in external repos doesn't see upstream updates within single # cli call, so we need to clean the caches to see the changes. clean_repos() - dvc.update("file.dvc") + tmp_dir.dvc.update("file.dvc") assert (tmp_dir / "file").read_text() == "second version" From 8bb2815ea8b9f3f5c0a822f1bcd8c3f81ae72671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Fri, 17 Jan 2020 09:50:50 +0000 Subject: [PATCH 4/5] simplify tests using new fixture --- tests/dir_helpers.py | 1 + tests/func/test_status.py | 30 +++++++++++++----------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 1e6d8ce308..8166c37923 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -307,6 +307,7 @@ def git_dir(tmp_path_factory, monkeypatch): try: path.scm = Git(fspath(path)) + path.scm.commit("initial commit to create the master branch") yield path finally: path.scm.close() diff --git a/tests/func/test_status.py b/tests/func/test_status.py index 8b8d3d2ba9..9c648f1011 100644 --- a/tests/func/test_status.py +++ b/tests/func/test_status.py @@ -61,27 +61,23 @@ def test_status_non_dvc_repo_import(tmp_dir, dvc, erepo_dir): } -def test_status_before_and_after_dvc_init(tmp_dir, dvc, erepo_dir): - with erepo_dir.chdir(): - erepo_dir.scm.repo.index.remove([".dvc"], r=True) - shutil.rmtree(".dvc") - erepo_dir.scm_gen("file", "first version") - erepo_dir.scm.add(["file"]) - erepo_dir.scm.commit("first version") - old_rev = erepo_dir.scm.get_rev() +def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir): + with git_dir.chdir(): + git_dir.scm_gen("file", "first version", commit="first version") + old_rev = git_dir.scm.get_rev() - dvc.imp(fspath(erepo_dir), "file", "file") + dvc.imp(fspath(git_dir), "file", "file") assert dvc.status(["file.dvc"]) == {} - with erepo_dir.chdir(): - Repo.init() - erepo_dir.scm.repo.index.remove(["file"]) + with git_dir.chdir(): + git_dir.dvc = Repo.init() + git_dir.scm.repo.index.remove(["file"]) os.remove("file") - erepo_dir.dvc_gen("file", "second version") - erepo_dir.scm.add([".dvc", "file.dvc"]) - erepo_dir.scm.commit("version with dvc") - new_rev = erepo_dir.scm.get_rev() + git_dir.dvc_gen("file", "second version") + git_dir.scm.add([".dvc", "file.dvc"]) + git_dir.scm.commit("version with dvc") + new_rev = git_dir.scm.get_rev() assert old_rev != new_rev @@ -93,6 +89,6 @@ def test_status_before_and_after_dvc_init(tmp_dir, dvc, erepo_dir): assert status == { "changed deps": { - "file ({})".format(fspath(erepo_dir)): "update available" + "file ({})".format(fspath(git_dir)): "update available" } } From 89e7aba5d5d0e2e912f2a702876ee63938787c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Fri, 17 Jan 2020 17:36:24 +0000 Subject: [PATCH 5/5] rename fixture --- tests/dir_helpers.py | 6 +++--- tests/func/test_status.py | 28 +++++++++++++++------------- tests/func/test_update.py | 14 ++++++++------ 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 8166c37923..a108d5b285 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -61,7 +61,7 @@ "repo_template", "run_copy", "erepo_dir", - "git_dir", + "external_git_dir", ] REPO_TEMPLATE = { "foo": "foo", @@ -296,10 +296,10 @@ def erepo_dir(tmp_path_factory, monkeypatch): @pytest.fixture -def git_dir(tmp_path_factory, monkeypatch): +def external_git_dir(tmp_path_factory, monkeypatch): from dvc.scm.git import Git - path = TmpDir(fspath_py35(tmp_path_factory.mktemp("git_dir"))) + path = TmpDir(fspath_py35(tmp_path_factory.mktemp("external_git_dir"))) # Create git repo with path.chdir(): diff --git a/tests/func/test_status.py b/tests/func/test_status.py index 9c648f1011..b9bca28a34 100644 --- a/tests/func/test_status.py +++ b/tests/func/test_status.py @@ -61,23 +61,25 @@ def test_status_non_dvc_repo_import(tmp_dir, dvc, erepo_dir): } -def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir): - with git_dir.chdir(): - git_dir.scm_gen("file", "first version", commit="first version") - old_rev = git_dir.scm.get_rev() +def test_status_before_and_after_dvc_init(tmp_dir, dvc, external_git_dir): + with external_git_dir.chdir(): + external_git_dir.scm_gen( + "file", "first version", commit="first version" + ) + old_rev = external_git_dir.scm.get_rev() - dvc.imp(fspath(git_dir), "file", "file") + dvc.imp(fspath(external_git_dir), "file", "file") assert dvc.status(["file.dvc"]) == {} - with git_dir.chdir(): - git_dir.dvc = Repo.init() - git_dir.scm.repo.index.remove(["file"]) + with external_git_dir.chdir(): + external_git_dir.dvc = Repo.init() + external_git_dir.scm.repo.index.remove(["file"]) os.remove("file") - git_dir.dvc_gen("file", "second version") - git_dir.scm.add([".dvc", "file.dvc"]) - git_dir.scm.commit("version with dvc") - new_rev = git_dir.scm.get_rev() + external_git_dir.dvc_gen("file", "second version") + external_git_dir.scm.add([".dvc", "file.dvc"]) + external_git_dir.scm.commit("version with dvc") + new_rev = external_git_dir.scm.get_rev() assert old_rev != new_rev @@ -89,6 +91,6 @@ def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir): assert status == { "changed deps": { - "file ({})".format(fspath(git_dir)): "update available" + "file ({})".format(fspath(external_git_dir)): "update available" } } diff --git a/tests/func/test_update.py b/tests/func/test_update.py index 6d8ae9da21..4e846a2bb2 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -174,19 +174,21 @@ def test_update_import_url(tmp_dir, dvc, tmp_path_factory): assert dst.read_text() == "updated file content" -def test_update_git_tracked(tmp_dir, dvc, git_dir): - with git_dir.chdir(): - git_dir.scm_gen("file", "first version", commit="create") +def test_update_git_tracked(tmp_dir, dvc, external_git_dir): + with external_git_dir.chdir(): + external_git_dir.scm_gen("file", "first version", commit="create") - tmp_dir.dvc.imp(fspath(git_dir), "file", "file") + tmp_dir.dvc.imp(fspath(external_git_dir), "file", "file") # Just to make sure it doesn't crash tmp_dir.dvc.update("file.dvc") assert (tmp_dir / "file").read_text() == "first version" - with git_dir.chdir(): - git_dir.scm_gen("file", "second version", commit="update file") + with external_git_dir.chdir(): + external_git_dir.scm_gen( + "file", "second version", commit="update file" + ) # Caching in external repos doesn't see upstream updates within single # cli call, so we need to clean the caches to see the changes.