From 14175c85b5f37934ee9cf32b9fa5975fb84ad286 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Mon, 19 Jul 2021 19:47:11 +0800 Subject: [PATCH 01/24] Push running checkpoint to remote Fix #6182 --- dvc/repo/experiments/executor/base.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index f797728042..d396087443 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -18,6 +18,7 @@ from dvc.exceptions import DvcException from dvc.path_info import PathInfo +from dvc.repo import Repo from dvc.repo.experiments.base import ( EXEC_BASELINE, EXEC_BRANCH, @@ -331,6 +332,7 @@ def filter_pipeline(stages): checkpoint_func = partial( cls.checkpoint_callback, + dvc, dvc.scm, name, repro_force or checkpoint_reset, @@ -393,7 +395,6 @@ def _repro_dvc( git_url: Optional[str] = None, **kwargs, ): - from dvc.repo import Repo from dvc.utils.serialize import modify_yaml dvc = Repo(dvc_dir) @@ -453,6 +454,7 @@ def _repro_args(cls, dvc): @classmethod def checkpoint_callback( cls, + dvc: "Repo", scm: "Git", name: Optional[str], force: bool, @@ -464,6 +466,23 @@ def checkpoint_callback( exp_rev = cls.commit( scm, exp_hash, exp_name=name, force=force, checkpoint=True ) + + git_remote = os.environ.get("DVC_EXP_AUTO_PUSH", None) + if git_remote: + from dvc.repo.experiments.push import push + + branch = dvc.experiments.get_branch_by_rev( + exp_rev, allow_multiple=None + ) + branch_name = ExpRefInfo.from_ref(branch).name + push( + dvc, + git_remote, + branch_name, + push_cache=True, + run_cache=True, + ) + logger.info({"pushed": branch_name}) logger.info("Checkpoint experiment iteration '%s'.", exp_rev[:7]) except UnchangedExperimentError: pass From 770e5bcb31a4a0915b1a363cb84db919986dc328 Mon Sep 17 00:00:00 2001 From: Gao Date: Tue, 20 Jul 2021 08:30:24 +0800 Subject: [PATCH 02/24] Update dvc/repo/experiments/executor/base.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Peter Rowlands (변기호) --- dvc/repo/experiments/executor/base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index d396087443..407e8e094e 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -471,10 +471,7 @@ def checkpoint_callback( if git_remote: from dvc.repo.experiments.push import push - branch = dvc.experiments.get_branch_by_rev( - exp_rev, allow_multiple=None - ) - branch_name = ExpRefInfo.from_ref(branch).name + branch = scm.get_ref(EXEC_BRANCH, follow=False) push( dvc, git_remote, From 15036f35201e34adbaee5bac6c38e77a3e27ba9d Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Wed, 21 Jul 2021 10:44:18 +0800 Subject: [PATCH 03/24] Get the branch name --- dvc/repo/experiments/executor/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 407e8e094e..84bc10a476 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -472,6 +472,7 @@ def checkpoint_callback( from dvc.repo.experiments.push import push branch = scm.get_ref(EXEC_BRANCH, follow=False) + branch_name = ExpRefInfo.from_ref(branch).name push( dvc, git_remote, From 205ce572a21516e6138d33cd8ba779f6640f42f3 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Mon, 26 Jul 2021 15:29:23 +0800 Subject: [PATCH 04/24] Finish this PR 1. move env name to dvc/env.py. 2. add some tests for it. --- dvc/env.py | 1 + dvc/repo/experiments/executor/base.py | 38 +++++++---- tests/func/experiments/conftest.py | 18 ++++++ tests/func/experiments/test_checkpoints.py | 74 ++++++++++++++++++++++ tests/func/experiments/test_remote.py | 18 ------ 5 files changed, 118 insertions(+), 31 deletions(-) diff --git a/dvc/env.py b/dvc/env.py index 0ac4e46e8b..15673e88da 100644 --- a/dvc/env.py +++ b/dvc/env.py @@ -7,3 +7,4 @@ DVCLIVE_HTML = "DVCLIVE_HTML" DVCLIVE_RESUME = "DVCLIVE_RESUME" DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY" +DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH" diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 84bc10a476..7dfc184ed0 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -16,6 +16,7 @@ from funcy import cached_property +from dvc.env import DVC_EXP_AUTO_PUSH from dvc.exceptions import DvcException from dvc.path_info import PathInfo from dvc.repo import Repo @@ -451,6 +452,28 @@ def _repro_args(cls, dvc): kwargs = {} return args, kwargs + @staticmethod + def _auto_push(git_remote: str, dvc: "Repo", scm: "Git"): + from dvc.repo.experiments.push import push + + if git_remote == scm.root_dir: + logger.warning( + f"{DVC_EXP_AUTO_PUSH} {git_remote} is the running " + "repository auto push will not work" + ) + return + + branch = scm.get_ref(EXEC_BRANCH, follow=False) + branch_name = ExpRefInfo.from_ref(branch).name + push( + dvc, + git_remote, + branch_name, + push_cache=True, + run_cache=True, + ) + logger.info({"pushed": branch_name}) + @classmethod def checkpoint_callback( cls, @@ -467,20 +490,9 @@ def checkpoint_callback( scm, exp_hash, exp_name=name, force=force, checkpoint=True ) - git_remote = os.environ.get("DVC_EXP_AUTO_PUSH", None) + git_remote = os.environ.get(DVC_EXP_AUTO_PUSH, None) if git_remote: - from dvc.repo.experiments.push import push - - branch = scm.get_ref(EXEC_BRANCH, follow=False) - branch_name = ExpRefInfo.from_ref(branch).name - push( - dvc, - git_remote, - branch_name, - push_cache=True, - run_cache=True, - ) - logger.info({"pushed": branch_name}) + cls._auto_push(git_remote, dvc, scm) logger.info("Checkpoint experiment iteration '%s'.", exp_rev[:7]) except UnchangedExperimentError: pass diff --git a/tests/func/experiments/conftest.py b/tests/func/experiments/conftest.py index 455aa687c4..fb592ae152 100644 --- a/tests/func/experiments/conftest.py +++ b/tests/func/experiments/conftest.py @@ -88,3 +88,21 @@ def checkpoint_stage(tmp_dir, scm, dvc, mocker): scm.commit("init") stage.iterations = DEFAULT_ITERATIONS return stage + + +@pytest.fixture +def git_upstream(tmp_dir, erepo_dir): + url = "file://{}".format(erepo_dir.resolve().as_posix()) + tmp_dir.scm.gitpython.repo.create_remote("upstream", url) + erepo_dir.remote = "upstream" + erepo_dir.url = url + return erepo_dir + + +@pytest.fixture +def git_downstream(tmp_dir, erepo_dir): + url = "file://{}".format(tmp_dir.resolve().as_posix()) + erepo_dir.scm.gitpython.repo.create_remote("upstream", url) + erepo_dir.remote = "upstream" + erepo_dir.url = url + return erepo_dir diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index 02c93cd14c..feb83ee82f 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -1,9 +1,13 @@ +import logging +import os + import pytest from funcy import first from dvc.exceptions import DvcException from dvc.repo.experiments import MultipleBranchError from dvc.repo.experiments.base import EXEC_APPLY, EXEC_CHECKPOINT +from dvc.repo.experiments.utils import exp_refs_by_rev @pytest.mark.parametrize("workspace", [True, False]) @@ -188,3 +192,73 @@ def test_resume_non_head_checkpoint( ) new_head = first(results) assert orig_branch != dvc.experiments.get_branch_by_rev(new_head) + + +@pytest.mark.parametrize("use_url", [True, False]) +def test_auto_push_during_iterations( + tmp_dir, scm, dvc, checkpoint_stage, git_upstream, local_remote, use_url +): + # set up remote repo + remote = git_upstream.url if use_url else git_upstream.remote + git_upstream.scm.fetch_refspecs(str(tmp_dir), ["master:master"]) + + # without auto push + results = dvc.experiments.run(checkpoint_stage.addressing) + exp = first(results) + ref_info = first(exp_refs_by_rev(scm, exp)) + assert git_upstream.scm.get_ref(str(ref_info)) is None + + # add auto push + os.environ["DVC_EXP_AUTO_PUSH"] = remote + results = dvc.experiments.run(checkpoint_stage.addressing) + assert (tmp_dir / "foo").read_text() == "4" + exp = first(results) + ref_info = first(exp_refs_by_rev(scm, exp)) + assert git_upstream.scm.get_ref(str(ref_info)) == exp + + # check the data + with git_upstream.dvc.config.edit() as conf: + conf["remote"]["local"] = local_remote.config + conf["core"]["remote"] = "local" + + git_upstream.dvc.experiments.apply(ref_info.name) + git_upstream.dvc.experiments.apply(exp) + git_upstream.dvc.pull() + assert (git_upstream / "foo").read_text() == "4" + + # resume the remote checkpoint + os.environ.pop("DVC_EXP_AUTO_PUSH") + with git_upstream.chdir(): + git_upstream.dvc.experiments.run(checkpoint_stage.addressing) + assert (git_upstream / "foo").read_text() == "6" + + +def test_auto_push_error_url(dvc, scm, checkpoint_stage, local_remote): + os.environ["DVC_EXP_AUTO_PUSH"] = "none" + assert ( + dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) + == {} + ) + + +def test_auto_push_no_remote(dvc, scm, checkpoint_stage, git_upstream): + os.environ["DVC_EXP_AUTO_PUSH"] = git_upstream.url + assert ( + dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) + == {} + ) + + +def test_auto_push_self_remote(tmp_dir, dvc, scm, checkpoint_stage, caplog): + root_dir = str(tmp_dir) + os.environ["DVC_EXP_AUTO_PUSH"] = root_dir + assert ( + dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) + != {} + ) + + with caplog.at_level(logging.WARNING, logger="dvc"): + assert ( + f"DVC_EXP_AUTO_PUSH {root_dir} is the running " + "repository auto push will not work" in caplog.messages + ) diff --git a/tests/func/experiments/test_remote.py b/tests/func/experiments/test_remote.py index 177476853b..dffb1970c6 100644 --- a/tests/func/experiments/test_remote.py +++ b/tests/func/experiments/test_remote.py @@ -7,24 +7,6 @@ from dvc.repo.experiments.utils import exp_refs_by_rev -@pytest.fixture -def git_upstream(tmp_dir, erepo_dir): - url = f"file://{erepo_dir.resolve().as_posix()}" - tmp_dir.scm.gitpython.repo.create_remote("upstream", url) - erepo_dir.remote = "upstream" - erepo_dir.url = url - return erepo_dir - - -@pytest.fixture -def git_downstream(tmp_dir, erepo_dir): - url = f"file://{tmp_dir.resolve().as_posix()}" - erepo_dir.scm.gitpython.repo.create_remote("upstream", url) - erepo_dir.remote = "upstream" - erepo_dir.url = url - return erepo_dir - - @pytest.mark.parametrize("use_url", [True, False]) def test_push(tmp_dir, scm, dvc, git_upstream, exp_stage, use_url): from dvc.exceptions import InvalidArgumentError From b6f147a3e7cc5e3c20eab96511d9a591f4b11691 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Tue, 27 Jul 2021 10:41:55 +0800 Subject: [PATCH 05/24] Update after finally commit --- dvc/repo/experiments/executor/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 7dfc184ed0..20c66cd1f6 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -364,6 +364,9 @@ def filter_pipeline(stages): force=repro_force, checkpoint=is_checkpoint, ) + git_remote = os.environ.get(DVC_EXP_AUTO_PUSH, None) + if git_remote: + cls._auto_push(git_remote, dvc, dvc.scm) except UnchangedExperimentError: pass ref = dvc.scm.get_ref(EXEC_BRANCH, follow=False) From 65eb8d6899436b70fe3d4493869fac05acd79dcd Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Tue, 27 Jul 2021 17:37:09 +0800 Subject: [PATCH 06/24] Some problems found in review 1. change the behaviour of self remote 2. do not use string DVC_EXP_AUTO_PUSH 3. downgrade the logger level in auto push 4. use full branch ref --- dvc/repo/experiments/executor/base.py | 16 ++++++---------- dvc/repo/experiments/push.py | 13 ++++++++++--- tests/func/experiments/test_checkpoints.py | 21 +++++++++++++-------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 20c66cd1f6..24427483f2 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -457,25 +457,21 @@ def _repro_args(cls, dvc): @staticmethod def _auto_push(git_remote: str, dvc: "Repo", scm: "Git"): - from dvc.repo.experiments.push import push - if git_remote == scm.root_dir: logger.warning( - f"{DVC_EXP_AUTO_PUSH} {git_remote} is the running " - "repository auto push will not work" + f"try to auto checkpoints to {git_remote} which is the " + "running repository dvc cache will be pushed to the " + "default remote while git references will not be pushed" ) - return branch = scm.get_ref(EXEC_BRANCH, follow=False) - branch_name = ExpRefInfo.from_ref(branch).name - push( - dvc, + dvc.experiments.push( git_remote, - branch_name, + branch, push_cache=True, run_cache=True, ) - logger.info({"pushed": branch_name}) + logger.debug(f"pushed {branch} to {git_remote}") @classmethod def checkpoint_callback( diff --git a/dvc/repo/experiments/push.py b/dvc/repo/experiments/push.py index ca0867bd8d..7b58682e33 100644 --- a/dvc/repo/experiments/push.py +++ b/dvc/repo/experiments/push.py @@ -2,6 +2,7 @@ from dvc.exceptions import DvcException, InvalidArgumentError from dvc.repo import locked +from dvc.repo.experiments.base import ExpRefInfo from dvc.repo.scm_context import scm_context from .utils import exp_commits, exp_refs_by_name @@ -12,7 +13,13 @@ @locked @scm_context def push( - repo, git_remote, exp_name, *args, force=False, push_cache=False, **kwargs + repo, + git_remote, + exp_name: str, + *args, + force=False, + push_cache=False, + **kwargs, ): exp_ref = _get_exp_ref(repo, exp_name) @@ -35,9 +42,9 @@ def on_diverged(refname: str, rev: str) -> bool: _push_cache(repo, exp_ref, **kwargs) -def _get_exp_ref(repo, exp_name): +def _get_exp_ref(repo, exp_name: str) -> ExpRefInfo: if exp_name.startswith("refs/"): - return exp_name + return ExpRefInfo.from_ref(exp_name) exp_refs = list(exp_refs_by_name(repo.scm, exp_name)) if not exp_refs: diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index feb83ee82f..d46d952d1b 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -4,6 +4,7 @@ import pytest from funcy import first +from dvc.env import DVC_EXP_AUTO_PUSH from dvc.exceptions import DvcException from dvc.repo.experiments import MultipleBranchError from dvc.repo.experiments.base import EXEC_APPLY, EXEC_CHECKPOINT @@ -209,7 +210,7 @@ def test_auto_push_during_iterations( assert git_upstream.scm.get_ref(str(ref_info)) is None # add auto push - os.environ["DVC_EXP_AUTO_PUSH"] = remote + os.environ[DVC_EXP_AUTO_PUSH] = remote results = dvc.experiments.run(checkpoint_stage.addressing) assert (tmp_dir / "foo").read_text() == "4" exp = first(results) @@ -227,14 +228,14 @@ def test_auto_push_during_iterations( assert (git_upstream / "foo").read_text() == "4" # resume the remote checkpoint - os.environ.pop("DVC_EXP_AUTO_PUSH") + os.environ.pop(DVC_EXP_AUTO_PUSH) with git_upstream.chdir(): git_upstream.dvc.experiments.run(checkpoint_stage.addressing) assert (git_upstream / "foo").read_text() == "6" def test_auto_push_error_url(dvc, scm, checkpoint_stage, local_remote): - os.environ["DVC_EXP_AUTO_PUSH"] = "none" + os.environ[DVC_EXP_AUTO_PUSH] = "none" assert ( dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) == {} @@ -242,16 +243,18 @@ def test_auto_push_error_url(dvc, scm, checkpoint_stage, local_remote): def test_auto_push_no_remote(dvc, scm, checkpoint_stage, git_upstream): - os.environ["DVC_EXP_AUTO_PUSH"] = git_upstream.url + os.environ[DVC_EXP_AUTO_PUSH] = git_upstream.url assert ( dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) == {} ) -def test_auto_push_self_remote(tmp_dir, dvc, scm, checkpoint_stage, caplog): +def test_auto_push_self_remote( + tmp_dir, dvc, scm, checkpoint_stage, local_remote, caplog +): root_dir = str(tmp_dir) - os.environ["DVC_EXP_AUTO_PUSH"] = root_dir + os.environ[DVC_EXP_AUTO_PUSH] = root_dir assert ( dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) != {} @@ -259,6 +262,8 @@ def test_auto_push_self_remote(tmp_dir, dvc, scm, checkpoint_stage, caplog): with caplog.at_level(logging.WARNING, logger="dvc"): assert ( - f"DVC_EXP_AUTO_PUSH {root_dir} is the running " - "repository auto push will not work" in caplog.messages + f"try to auto checkpoints to {root_dir} which is the " + "running repository dvc cache will be pushed to the " + "default remote while git references will not be pushed" + in caplog.text ) From 507786256b413a8ea88609467249b1b6c16bad3c Mon Sep 17 00:00:00 2001 From: Gao Date: Tue, 27 Jul 2021 21:34:42 +0800 Subject: [PATCH 07/24] Update dvc/repo/experiments/executor/base.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Peter Rowlands (변기호) --- dvc/repo/experiments/executor/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 24427483f2..e0b334a104 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -471,7 +471,6 @@ def _auto_push(git_remote: str, dvc: "Repo", scm: "Git"): push_cache=True, run_cache=True, ) - logger.debug(f"pushed {branch} to {git_remote}") @classmethod def checkpoint_callback( From dd7b380c16bc73137862c5465535c05e88aedb86 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Mon, 2 Aug 2021 15:29:11 +0800 Subject: [PATCH 08/24] add remote check before auto push --- dvc/repo/experiments/executor/base.py | 32 +++++++++++++++++----- dvc/scm/git/backend/dulwich.py | 18 ++++++------ tests/func/experiments/test_checkpoints.py | 20 ++++++-------- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index e0b334a104..bd8434cb11 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -250,6 +250,28 @@ def on_diverged_ref(orig_ref: str, new_rev: str): ) return refs + @classmethod + def _auto_push_check(cls, dvc: "Repo"): + git_remote = os.environ.get(DVC_EXP_AUTO_PUSH, None) + if git_remote == dvc.root_dir: + logger.warning( + f"'{git_remote}' points to the current Git repo, experiment " + "Git refs will not be pushed. But DVC cache and run cache " + "will automatically be pushed to the default DVC remote " + "(if any) on each experiment commit." + ) + try: + for ref in dvc.scm.iter_remote_refs( + git_remote, base=EXPS_NAMESPACE + ): + if ref: + break + except BaseException as e: + print(e) + raise e + + dvc.cloud.get_remote_odb() + @classmethod def reproduce( cls, @@ -294,6 +316,9 @@ def filter_pipeline(stages): log_errors, **kwargs, ) as dvc: + if os.environ.get(DVC_EXP_AUTO_PUSH, None): + cls._auto_push_check(dvc) + args, kwargs = cls._repro_args(dvc) if args: targets: Optional[Union[list, str]] = args[0] @@ -457,13 +482,6 @@ def _repro_args(cls, dvc): @staticmethod def _auto_push(git_remote: str, dvc: "Repo", scm: "Git"): - if git_remote == scm.root_dir: - logger.warning( - f"try to auto checkpoints to {git_remote} which is the " - "running repository dvc cache will be pushed to the " - "default remote while git references will not be pushed" - ) - branch = scm.get_ref(EXEC_BRANCH, follow=False) dvc.experiments.push( git_remote, diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index c1860f85ea..06bad6e7cc 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -353,20 +353,20 @@ def iter_remote_refs(self, url: str, base: Optional[str] = None): try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location) + if base: + yield from ( + os.fsdecode(ref) + for ref in client.get_refs(path) + if ref.startswith(os.fsencode(base)) + ) + else: + yield from (os.fsdecode(ref) for ref in client.get_refs(path)) + except Exception as exc: raise SCMError( f"'{url}' is not a valid Git remote or URL" ) from exc - if base: - yield from ( - os.fsdecode(ref) - for ref in client.get_refs(path) - if ref.startswith(os.fsencode(base)) - ) - else: - yield from (os.fsdecode(ref) for ref in client.get_refs(path)) - def get_refs_containing(self, rev: str, pattern: Optional[str] = None): raise NotImplementedError diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index d46d952d1b..0cecc04a19 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -4,11 +4,13 @@ import pytest from funcy import first +from dvc.config import NoRemoteError from dvc.env import DVC_EXP_AUTO_PUSH from dvc.exceptions import DvcException from dvc.repo.experiments import MultipleBranchError from dvc.repo.experiments.base import EXEC_APPLY, EXEC_CHECKPOINT from dvc.repo.experiments.utils import exp_refs_by_rev +from dvc.scm.base import SCMError @pytest.mark.parametrize("workspace", [True, False]) @@ -236,18 +238,14 @@ def test_auto_push_during_iterations( def test_auto_push_error_url(dvc, scm, checkpoint_stage, local_remote): os.environ[DVC_EXP_AUTO_PUSH] = "none" - assert ( + with pytest.raises(SCMError): dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) - == {} - ) def test_auto_push_no_remote(dvc, scm, checkpoint_stage, git_upstream): os.environ[DVC_EXP_AUTO_PUSH] = git_upstream.url - assert ( + with pytest.raises(NoRemoteError): dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) - == {} - ) def test_auto_push_self_remote( @@ -260,10 +258,10 @@ def test_auto_push_self_remote( != {} ) - with caplog.at_level(logging.WARNING, logger="dvc"): + with caplog.at_level(logging.WARNING, logger="dvc.repo.experiments"): assert ( - f"try to auto checkpoints to {root_dir} which is the " - "running repository dvc cache will be pushed to the " - "default remote while git references will not be pushed" - in caplog.text + f"'{root_dir}' points to the current Git repo, experiment " + "Git refs will not be pushed. But DVC cache and run cache will " + "automatically be pushed to the default DVC remote (if any) " + "on each experiment commit." in caplog.text ) From 92d9b07f999cdbb6fb694bb7dfc148ed0a512e18 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Wed, 4 Aug 2021 16:21:49 +0800 Subject: [PATCH 09/24] Split current one env into two --- dvc/env.py | 3 +- dvc/repo/experiments/executor/base.py | 21 +++++---- dvc/utils/__init__.py | 9 ++++ tests/func/experiments/test_checkpoints.py | 51 +++++++++++++++++----- tests/unit/utils/test_utils.py | 20 +++++++++ 5 files changed, 81 insertions(+), 23 deletions(-) diff --git a/dvc/env.py b/dvc/env.py index 15673e88da..adc83bff9f 100644 --- a/dvc/env.py +++ b/dvc/env.py @@ -7,4 +7,5 @@ DVCLIVE_HTML = "DVCLIVE_HTML" DVCLIVE_RESUME = "DVCLIVE_RESUME" DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY" -DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH" +DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE" +DVC_EXP_CHECKPOINT_PUSH = "DVC_EXP_CHECKPOINT_PUSH" diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index bd8434cb11..9e288e9fdf 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -16,7 +16,7 @@ from funcy import cached_property -from dvc.env import DVC_EXP_AUTO_PUSH +from dvc.env import DVC_EXP_CHECKPOINT_PUSH, DVC_EXP_GIT_REMOTE from dvc.exceptions import DvcException from dvc.path_info import PathInfo from dvc.repo import Repo @@ -38,7 +38,7 @@ from dvc.stage import PipelineStage from dvc.stage.monitor import CheckpointKilledError from dvc.stage.serialize import to_lockfile -from dvc.utils import dict_sha256 +from dvc.utils import dict_sha256, getenv_bool from dvc.utils.fs import remove if TYPE_CHECKING: @@ -252,7 +252,7 @@ def on_diverged_ref(orig_ref: str, new_rev: str): @classmethod def _auto_push_check(cls, dvc: "Repo"): - git_remote = os.environ.get(DVC_EXP_AUTO_PUSH, None) + git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None) if git_remote == dvc.root_dir: logger.warning( f"'{git_remote}' points to the current Git repo, experiment " @@ -316,7 +316,7 @@ def filter_pipeline(stages): log_errors, **kwargs, ) as dvc: - if os.environ.get(DVC_EXP_AUTO_PUSH, None): + if getenv_bool(DVC_EXP_CHECKPOINT_PUSH): cls._auto_push_check(dvc) args, kwargs = cls._repro_args(dvc) @@ -389,9 +389,8 @@ def filter_pipeline(stages): force=repro_force, checkpoint=is_checkpoint, ) - git_remote = os.environ.get(DVC_EXP_AUTO_PUSH, None) - if git_remote: - cls._auto_push(git_remote, dvc, dvc.scm) + if getenv_bool(DVC_EXP_CHECKPOINT_PUSH): + cls._auto_push(dvc, dvc.scm) except UnchangedExperimentError: pass ref = dvc.scm.get_ref(EXEC_BRANCH, follow=False) @@ -481,7 +480,8 @@ def _repro_args(cls, dvc): return args, kwargs @staticmethod - def _auto_push(git_remote: str, dvc: "Repo", scm: "Git"): + def _auto_push(dvc: "Repo", scm: "Git"): + git_remote = os.getenv(DVC_EXP_GIT_REMOTE) branch = scm.get_ref(EXEC_BRANCH, follow=False) dvc.experiments.push( git_remote, @@ -506,9 +506,8 @@ def checkpoint_callback( scm, exp_hash, exp_name=name, force=force, checkpoint=True ) - git_remote = os.environ.get(DVC_EXP_AUTO_PUSH, None) - if git_remote: - cls._auto_push(git_remote, dvc, scm) + if getenv_bool(DVC_EXP_CHECKPOINT_PUSH): + cls._auto_push(dvc, scm) logger.info("Checkpoint experiment iteration '%s'.", exp_rev[:7]) except UnchangedExperimentError: pass diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index e396be44b5..dcf072f347 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -1,5 +1,6 @@ """Helpers for other modules.""" +import ast import hashlib import json import logging @@ -512,3 +513,11 @@ def errored_revisions(rev_data: Dict) -> List: if nested_contains(data, "error"): result.append(revision) return result + + +def getenv_bool(name: str, default: str = "False"): + raw = os.getenv(name, default).title() + try: + return bool(ast.literal_eval(raw)) + except ValueError: + return False diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index 0cecc04a19..ea27f80943 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -1,11 +1,10 @@ import logging -import os import pytest from funcy import first from dvc.config import NoRemoteError -from dvc.env import DVC_EXP_AUTO_PUSH +from dvc.env import DVC_EXP_CHECKPOINT_PUSH, DVC_EXP_GIT_REMOTE from dvc.exceptions import DvcException from dvc.repo.experiments import MultipleBranchError from dvc.repo.experiments.base import EXEC_APPLY, EXEC_CHECKPOINT @@ -197,13 +196,29 @@ def test_resume_non_head_checkpoint( assert orig_branch != dvc.experiments.get_branch_by_rev(new_head) +@pytest.fixture +def clear_env(monkeypatch): + yield + monkeypatch.delenv(DVC_EXP_GIT_REMOTE, raising=False) + monkeypatch.delenv(DVC_EXP_CHECKPOINT_PUSH, raising=False) + + @pytest.mark.parametrize("use_url", [True, False]) def test_auto_push_during_iterations( - tmp_dir, scm, dvc, checkpoint_stage, git_upstream, local_remote, use_url + tmp_dir, + scm, + dvc, + checkpoint_stage, + git_upstream, + local_remote, + use_url, + monkeypatch, + clear_env, ): # set up remote repo remote = git_upstream.url if use_url else git_upstream.remote git_upstream.scm.fetch_refspecs(str(tmp_dir), ["master:master"]) + monkeypatch.setenv(DVC_EXP_GIT_REMOTE, remote) # without auto push results = dvc.experiments.run(checkpoint_stage.addressing) @@ -212,7 +227,7 @@ def test_auto_push_during_iterations( assert git_upstream.scm.get_ref(str(ref_info)) is None # add auto push - os.environ[DVC_EXP_AUTO_PUSH] = remote + monkeypatch.setenv(DVC_EXP_CHECKPOINT_PUSH, "true") results = dvc.experiments.run(checkpoint_stage.addressing) assert (tmp_dir / "foo").read_text() == "4" exp = first(results) @@ -230,29 +245,43 @@ def test_auto_push_during_iterations( assert (git_upstream / "foo").read_text() == "4" # resume the remote checkpoint - os.environ.pop(DVC_EXP_AUTO_PUSH) + monkeypatch.delenv(DVC_EXP_CHECKPOINT_PUSH, raising=False) with git_upstream.chdir(): git_upstream.dvc.experiments.run(checkpoint_stage.addressing) assert (git_upstream / "foo").read_text() == "6" -def test_auto_push_error_url(dvc, scm, checkpoint_stage, local_remote): - os.environ[DVC_EXP_AUTO_PUSH] = "none" +def test_auto_push_error_url( + dvc, scm, checkpoint_stage, local_remote, monkeypatch, clear_env +): + monkeypatch.setenv(DVC_EXP_GIT_REMOTE, "true") + monkeypatch.setenv(DVC_EXP_CHECKPOINT_PUSH, "true") with pytest.raises(SCMError): dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) -def test_auto_push_no_remote(dvc, scm, checkpoint_stage, git_upstream): - os.environ[DVC_EXP_AUTO_PUSH] = git_upstream.url +def test_auto_push_no_remote( + dvc, scm, checkpoint_stage, git_upstream, monkeypatch, clear_env +): + monkeypatch.setenv(DVC_EXP_GIT_REMOTE, git_upstream.url) + monkeypatch.setenv(DVC_EXP_CHECKPOINT_PUSH, "true") with pytest.raises(NoRemoteError): dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) def test_auto_push_self_remote( - tmp_dir, dvc, scm, checkpoint_stage, local_remote, caplog + tmp_dir, + dvc, + scm, + checkpoint_stage, + local_remote, + caplog, + monkeypatch, + clear_env, ): root_dir = str(tmp_dir) - os.environ[DVC_EXP_AUTO_PUSH] = root_dir + monkeypatch.setenv(DVC_EXP_GIT_REMOTE, root_dir) + monkeypatch.setenv(DVC_EXP_CHECKPOINT_PUSH, "true") assert ( dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) != {} diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 43d9c5ba38..df1152c1c8 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -9,6 +9,7 @@ dict_sha256, file_md5, fix_env, + getenv_bool, parse_target, relpath, resolve_output, @@ -232,3 +233,22 @@ def test_hint_on_lockfile(): ) def test_dict_sha256(d, sha): assert dict_sha256(d) == sha + + +@pytest.mark.parametrize( + "value, result", + [ + ("true", True), + ("True", True), + ("1", True), + ("false", False), + ("False", False), + ("0", False), + ("other", False), + ], +) +def test_getenv_bool(monkeypatch, value, result): + env = "env" + monkeypatch.setenv(env, value) + assert getenv_bool(env) == result + monkeypatch.delenv(env) From 7c271b3577d488462841e153b28a9e35ae268d48 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Thu, 5 Aug 2021 16:00:07 +0800 Subject: [PATCH 10/24] Rename DVC_EXP_CHECKPOINT_PUSH to DVC_EXP_AUTO_PUSH --- dvc/env.py | 2 +- dvc/repo/experiments/executor/base.py | 8 ++++---- tests/func/experiments/test_checkpoints.py | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dvc/env.py b/dvc/env.py index adc83bff9f..96d97fea9e 100644 --- a/dvc/env.py +++ b/dvc/env.py @@ -8,4 +8,4 @@ DVCLIVE_RESUME = "DVCLIVE_RESUME" DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY" DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE" -DVC_EXP_CHECKPOINT_PUSH = "DVC_EXP_CHECKPOINT_PUSH" +DVC_EXP_AUTO_PUSH = "DVC_EXP_CHECKPOINT_PUSH" diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 9e288e9fdf..43eb5241da 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -16,7 +16,7 @@ from funcy import cached_property -from dvc.env import DVC_EXP_CHECKPOINT_PUSH, DVC_EXP_GIT_REMOTE +from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE from dvc.exceptions import DvcException from dvc.path_info import PathInfo from dvc.repo import Repo @@ -316,7 +316,7 @@ def filter_pipeline(stages): log_errors, **kwargs, ) as dvc: - if getenv_bool(DVC_EXP_CHECKPOINT_PUSH): + if getenv_bool(DVC_EXP_AUTO_PUSH): cls._auto_push_check(dvc) args, kwargs = cls._repro_args(dvc) @@ -389,7 +389,7 @@ def filter_pipeline(stages): force=repro_force, checkpoint=is_checkpoint, ) - if getenv_bool(DVC_EXP_CHECKPOINT_PUSH): + if getenv_bool(DVC_EXP_AUTO_PUSH): cls._auto_push(dvc, dvc.scm) except UnchangedExperimentError: pass @@ -506,7 +506,7 @@ def checkpoint_callback( scm, exp_hash, exp_name=name, force=force, checkpoint=True ) - if getenv_bool(DVC_EXP_CHECKPOINT_PUSH): + if getenv_bool(DVC_EXP_AUTO_PUSH): cls._auto_push(dvc, scm) logger.info("Checkpoint experiment iteration '%s'.", exp_rev[:7]) except UnchangedExperimentError: diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index ea27f80943..101da1a9f7 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -4,7 +4,7 @@ from funcy import first from dvc.config import NoRemoteError -from dvc.env import DVC_EXP_CHECKPOINT_PUSH, DVC_EXP_GIT_REMOTE +from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE from dvc.exceptions import DvcException from dvc.repo.experiments import MultipleBranchError from dvc.repo.experiments.base import EXEC_APPLY, EXEC_CHECKPOINT @@ -200,7 +200,7 @@ def test_resume_non_head_checkpoint( def clear_env(monkeypatch): yield monkeypatch.delenv(DVC_EXP_GIT_REMOTE, raising=False) - monkeypatch.delenv(DVC_EXP_CHECKPOINT_PUSH, raising=False) + monkeypatch.delenv(DVC_EXP_AUTO_PUSH, raising=False) @pytest.mark.parametrize("use_url", [True, False]) @@ -227,7 +227,7 @@ def test_auto_push_during_iterations( assert git_upstream.scm.get_ref(str(ref_info)) is None # add auto push - monkeypatch.setenv(DVC_EXP_CHECKPOINT_PUSH, "true") + monkeypatch.setenv(DVC_EXP_AUTO_PUSH, "true") results = dvc.experiments.run(checkpoint_stage.addressing) assert (tmp_dir / "foo").read_text() == "4" exp = first(results) @@ -245,7 +245,7 @@ def test_auto_push_during_iterations( assert (git_upstream / "foo").read_text() == "4" # resume the remote checkpoint - monkeypatch.delenv(DVC_EXP_CHECKPOINT_PUSH, raising=False) + monkeypatch.delenv(DVC_EXP_AUTO_PUSH, raising=False) with git_upstream.chdir(): git_upstream.dvc.experiments.run(checkpoint_stage.addressing) assert (git_upstream / "foo").read_text() == "6" @@ -255,7 +255,7 @@ def test_auto_push_error_url( dvc, scm, checkpoint_stage, local_remote, monkeypatch, clear_env ): monkeypatch.setenv(DVC_EXP_GIT_REMOTE, "true") - monkeypatch.setenv(DVC_EXP_CHECKPOINT_PUSH, "true") + monkeypatch.setenv(DVC_EXP_AUTO_PUSH, "true") with pytest.raises(SCMError): dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) @@ -264,7 +264,7 @@ def test_auto_push_no_remote( dvc, scm, checkpoint_stage, git_upstream, monkeypatch, clear_env ): monkeypatch.setenv(DVC_EXP_GIT_REMOTE, git_upstream.url) - monkeypatch.setenv(DVC_EXP_CHECKPOINT_PUSH, "true") + monkeypatch.setenv(DVC_EXP_AUTO_PUSH, "true") with pytest.raises(NoRemoteError): dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) @@ -281,7 +281,7 @@ def test_auto_push_self_remote( ): root_dir = str(tmp_dir) monkeypatch.setenv(DVC_EXP_GIT_REMOTE, root_dir) - monkeypatch.setenv(DVC_EXP_CHECKPOINT_PUSH, "true") + monkeypatch.setenv(DVC_EXP_AUTO_PUSH, "true") assert ( dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) != {} From 34f4da33ad442aae9f1101dfbd4d13c6f4cfc28e Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Thu, 5 Aug 2021 21:36:50 +0800 Subject: [PATCH 11/24] value error --- dvc/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/env.py b/dvc/env.py index 96d97fea9e..f60cffdc58 100644 --- a/dvc/env.py +++ b/dvc/env.py @@ -8,4 +8,4 @@ DVCLIVE_RESUME = "DVCLIVE_RESUME" DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY" DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE" -DVC_EXP_AUTO_PUSH = "DVC_EXP_CHECKPOINT_PUSH" +DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH " From 37d2150557d6f3491898df029941470104166f29 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Fri, 6 Aug 2021 11:06:01 +0800 Subject: [PATCH 12/24] Remove getenv_bool switch to env2bool --- dvc/repo/experiments/executor/base.py | 8 ++++---- dvc/utils/__init__.py | 9 --------- tests/unit/utils/test_utils.py | 20 -------------------- 3 files changed, 4 insertions(+), 33 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 43eb5241da..da937b62ca 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -38,7 +38,7 @@ from dvc.stage import PipelineStage from dvc.stage.monitor import CheckpointKilledError from dvc.stage.serialize import to_lockfile -from dvc.utils import dict_sha256, getenv_bool +from dvc.utils import dict_sha256, env2bool from dvc.utils.fs import remove if TYPE_CHECKING: @@ -316,7 +316,7 @@ def filter_pipeline(stages): log_errors, **kwargs, ) as dvc: - if getenv_bool(DVC_EXP_AUTO_PUSH): + if env2bool(DVC_EXP_AUTO_PUSH): cls._auto_push_check(dvc) args, kwargs = cls._repro_args(dvc) @@ -389,7 +389,7 @@ def filter_pipeline(stages): force=repro_force, checkpoint=is_checkpoint, ) - if getenv_bool(DVC_EXP_AUTO_PUSH): + if env2bool(DVC_EXP_AUTO_PUSH): cls._auto_push(dvc, dvc.scm) except UnchangedExperimentError: pass @@ -506,7 +506,7 @@ def checkpoint_callback( scm, exp_hash, exp_name=name, force=force, checkpoint=True ) - if getenv_bool(DVC_EXP_AUTO_PUSH): + if env2bool(DVC_EXP_AUTO_PUSH): cls._auto_push(dvc, scm) logger.info("Checkpoint experiment iteration '%s'.", exp_rev[:7]) except UnchangedExperimentError: diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index dcf072f347..e396be44b5 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -1,6 +1,5 @@ """Helpers for other modules.""" -import ast import hashlib import json import logging @@ -513,11 +512,3 @@ def errored_revisions(rev_data: Dict) -> List: if nested_contains(data, "error"): result.append(revision) return result - - -def getenv_bool(name: str, default: str = "False"): - raw = os.getenv(name, default).title() - try: - return bool(ast.literal_eval(raw)) - except ValueError: - return False diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index df1152c1c8..43d9c5ba38 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -9,7 +9,6 @@ dict_sha256, file_md5, fix_env, - getenv_bool, parse_target, relpath, resolve_output, @@ -233,22 +232,3 @@ def test_hint_on_lockfile(): ) def test_dict_sha256(d, sha): assert dict_sha256(d) == sha - - -@pytest.mark.parametrize( - "value, result", - [ - ("true", True), - ("True", True), - ("1", True), - ("false", False), - ("False", False), - ("0", False), - ("other", False), - ], -) -def test_getenv_bool(monkeypatch, value, result): - env = "env" - monkeypatch.setenv(env, value) - assert getenv_bool(env) == result - monkeypatch.delenv(env) From 0cfd8d15557b938b5b1b24624104226749ab131a Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Fri, 6 Aug 2021 17:11:10 +0800 Subject: [PATCH 13/24] Better on handling dulwich exceptions --- dvc/scm/git/backend/dulwich.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 06bad6e7cc..3f8482067f 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -348,11 +348,18 @@ def iter_refs(self, base: Optional[str] = None): def iter_remote_refs(self, url: str, base: Optional[str] = None): from dulwich.client import get_transport_and_path + from dulwich.errors import NotGitRepository from dulwich.porcelain import get_remote_repo try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location) + except Exception as exc: + raise SCMError( + f"'{url}' is not a valid Git remote or URL" + ) from exc + + try: if base: yield from ( os.fsdecode(ref) @@ -361,11 +368,8 @@ def iter_remote_refs(self, url: str, base: Optional[str] = None): ) else: yield from (os.fsdecode(ref) for ref in client.get_refs(path)) - - except Exception as exc: - raise SCMError( - f"'{url}' is not a valid Git remote or URL" - ) from exc + except NotGitRepository as exc: + raise SCMError(f"{url} is not a git repository") from exc def get_refs_containing(self, rev: str, pattern: Optional[str] = None): raise NotImplementedError From ce7cd7d3f6659bf99a4ba0b42ebf15d2f2409cd5 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Fri, 6 Aug 2021 17:23:55 +0800 Subject: [PATCH 14/24] Name changing removing prints --- dvc/repo/experiments/executor/base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index da937b62ca..feff27619d 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -251,7 +251,7 @@ def on_diverged_ref(orig_ref: str, new_rev: str): return refs @classmethod - def _auto_push_check(cls, dvc: "Repo"): + def _validate_remotes(cls, dvc: "Repo"): git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None) if git_remote == dvc.root_dir: logger.warning( @@ -267,7 +267,6 @@ def _auto_push_check(cls, dvc: "Repo"): if ref: break except BaseException as e: - print(e) raise e dvc.cloud.get_remote_odb() @@ -317,7 +316,7 @@ def filter_pipeline(stages): **kwargs, ) as dvc: if env2bool(DVC_EXP_AUTO_PUSH): - cls._auto_push_check(dvc) + cls._validate_remotes(dvc) args, kwargs = cls._repro_args(dvc) if args: From e666d2e614f0d4adfef9cdb548d9877f32eda55a Mon Sep 17 00:00:00 2001 From: Gao Date: Mon, 9 Aug 2021 09:48:30 +0800 Subject: [PATCH 15/24] Update dvc/env.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Peter Rowlands (변기호) --- dvc/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/env.py b/dvc/env.py index f60cffdc58..e5f8ad912b 100644 --- a/dvc/env.py +++ b/dvc/env.py @@ -8,4 +8,4 @@ DVCLIVE_RESUME = "DVCLIVE_RESUME" DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY" DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE" -DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH " +DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH" From 50d419f056eff4c7741123e1221cab1d60d4f9ce Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Mon, 9 Aug 2021 21:56:40 +0800 Subject: [PATCH 16/24] Some changes in review --- dvc/repo/experiments/executor/base.py | 62 +++++++++++++++++---------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index feff27619d..d2707f5489 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -251,8 +251,12 @@ def on_diverged_ref(orig_ref: str, new_rev: str): return refs @classmethod - def _validate_remotes(cls, dvc: "Repo"): - git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None) + def _validate_remotes(cls, dvc: "Repo", git_remote: Optional[str]): + from dulwich.client import get_transport_and_path + from dulwich.porcelain import get_remote_repo + + from dvc.scm.base import SCMError + if git_remote == dvc.root_dir: logger.warning( f"'{git_remote}' points to the current Git repo, experiment " @@ -261,13 +265,12 @@ def _validate_remotes(cls, dvc: "Repo"): "(if any) on each experiment commit." ) try: - for ref in dvc.scm.iter_remote_refs( - git_remote, base=EXPS_NAMESPACE - ): - if ref: - break - except BaseException as e: - raise e + _, location = get_remote_repo(dvc.scm.dulwich.repo, git_remote) + get_transport_and_path(location) + except Exception as exc: + raise SCMError( + f"'{git_remote}' is not a valid Git remote or URL" + ) from exc dvc.cloud.get_remote_odb() @@ -293,6 +296,9 @@ def reproduce( from dvc.repo.checkout import checkout as dvc_checkout from dvc.repo.reproduce import reproduce as dvc_reproduce + auto_push = env2bool(DVC_EXP_AUTO_PUSH) + git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None) + unchanged = [] if queue is not None: @@ -315,8 +321,8 @@ def filter_pipeline(stages): log_errors, **kwargs, ) as dvc: - if env2bool(DVC_EXP_AUTO_PUSH): - cls._validate_remotes(dvc) + if auto_push: + cls._validate_remotes(dvc, git_remote) args, kwargs = cls._repro_args(dvc) if args: @@ -388,8 +394,8 @@ def filter_pipeline(stages): force=repro_force, checkpoint=is_checkpoint, ) - if env2bool(DVC_EXP_AUTO_PUSH): - cls._auto_push(dvc, dvc.scm) + if auto_push: + cls._auto_push(dvc, dvc.scm, git_remote) except UnchangedExperimentError: pass ref = dvc.scm.get_ref(EXEC_BRANCH, follow=False) @@ -479,15 +485,26 @@ def _repro_args(cls, dvc): return args, kwargs @staticmethod - def _auto_push(dvc: "Repo", scm: "Git"): - git_remote = os.getenv(DVC_EXP_GIT_REMOTE) + def _auto_push( + dvc: "Repo", + scm: "Git", + git_remote: Optional[str], + push_cache=True, + run_cache=True, + ): branch = scm.get_ref(EXEC_BRANCH, follow=False) - dvc.experiments.push( - git_remote, - branch, - push_cache=True, - run_cache=True, - ) + try: + dvc.experiments.push( + git_remote, + branch, + push_cache=push_cache, + run_cache=run_cache, + ) + except BaseException: + logger.warning( + "Something went wrong while auto pushing checkpoint cache " + f"to the remote {git_remote}" + ) @classmethod def checkpoint_callback( @@ -506,7 +523,8 @@ def checkpoint_callback( ) if env2bool(DVC_EXP_AUTO_PUSH): - cls._auto_push(dvc, scm) + git_remote = os.getenv(DVC_EXP_GIT_REMOTE) + cls._auto_push(dvc, scm, git_remote) logger.info("Checkpoint experiment iteration '%s'.", exp_rev[:7]) except UnchangedExperimentError: pass From 5cf541cf2442dbde582f921f22f37cc0c1045dc3 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Tue, 10 Aug 2021 11:44:31 +0800 Subject: [PATCH 17/24] Add validation to repo url --- dvc/repo/experiments/executor/base.py | 13 +------------ dvc/scm/git/__init__.py | 1 + dvc/scm/git/backend/base.py | 4 ++++ dvc/scm/git/backend/dulwich.py | 14 ++++++++++++++ dvc/scm/git/backend/gitpython.py | 3 +++ dvc/scm/git/backend/pygit2.py | 3 +++ 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index d2707f5489..54c1402246 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -252,10 +252,6 @@ def on_diverged_ref(orig_ref: str, new_rev: str): @classmethod def _validate_remotes(cls, dvc: "Repo", git_remote: Optional[str]): - from dulwich.client import get_transport_and_path - from dulwich.porcelain import get_remote_repo - - from dvc.scm.base import SCMError if git_remote == dvc.root_dir: logger.warning( @@ -264,14 +260,7 @@ def _validate_remotes(cls, dvc: "Repo", git_remote: Optional[str]): "will automatically be pushed to the default DVC remote " "(if any) on each experiment commit." ) - try: - _, location = get_remote_repo(dvc.scm.dulwich.repo, git_remote) - get_transport_and_path(location) - except Exception as exc: - raise SCMError( - f"'{git_remote}' is not a valid Git remote or URL" - ) from exc - + dvc.scm.validate_git_repo(git_remote) dvc.cloud.get_remote_odb() @classmethod diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 8be455a546..629038228f 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -392,6 +392,7 @@ def get_fs(self, rev: str): checkout_index = partialmethod(_backend_func, "checkout_index") status = partialmethod(_backend_func, "status") merge = partialmethod(_backend_func, "merge") + validate_git_repo = partialmethod(_backend_func, "validate_git_repo") def resolve_rev(self, rev: str) -> str: from dvc.repo.experiments.utils import exp_refs_by_name diff --git a/dvc/scm/git/backend/base.py b/dvc/scm/git/backend/base.py index 3196f35907..bb11390e8d 100644 --- a/dvc/scm/git/backend/base.py +++ b/dvc/scm/git/backend/base.py @@ -347,3 +347,7 @@ def merge( Returns revision of the merge commit or None if no commit was made. """ + + @abstractmethod + def validate_git_repo(self, url: str): + """validate remote git repo""" diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 3f8482067f..2b37e2a8e3 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -646,3 +646,17 @@ def merge( squash: bool = False, ) -> Optional[str]: raise NotImplementedError + + def validate_git_repo(self, url: str): + from dulwich.client import LocalGitClient, get_transport_and_path + from dulwich.porcelain import get_remote_repo + + try: + _, location = get_remote_repo(self.repo, url) + client, path = get_transport_and_path(location) + except Exception as exc: + raise SCMError( + f"'{url}' is not a valid Git remote or URL" + ) from exc + if isinstance(client, LocalGitClient) and not os.path.exists(path): + raise SCMError(f"'{url}' is not a valid Git remote or URL") diff --git a/dvc/scm/git/backend/gitpython.py b/dvc/scm/git/backend/gitpython.py index c49ee8ef7c..3ba57ced7a 100644 --- a/dvc/scm/git/backend/gitpython.py +++ b/dvc/scm/git/backend/gitpython.py @@ -618,3 +618,6 @@ def merge( raise MergeConflictError("Merge contained conflicts") from exc raise SCMError("Merge failed") from exc return None + + def validate_git_repo(self, url: str): + raise NotImplementedError diff --git a/dvc/scm/git/backend/pygit2.py b/dvc/scm/git/backend/pygit2.py index aeee30579b..6c3603b3c6 100644 --- a/dvc/scm/git/backend/pygit2.py +++ b/dvc/scm/git/backend/pygit2.py @@ -561,3 +561,6 @@ def merge( self.repo.state_cleanup() self.repo.index.write() return None + + def validate_git_repo(self, url: str): + raise NotImplementedError From dfcc94646f4659f9a8cad91d8acd6a43040dce9a Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Tue, 10 Aug 2021 14:57:20 +0800 Subject: [PATCH 18/24] Add new type of SCMError --- dvc/scm/base.py | 6 ++++++ dvc/scm/git/backend/dulwich.py | 14 +++++--------- tests/func/experiments/test_checkpoints.py | 4 ++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/dvc/scm/base.py b/dvc/scm/base.py index cf7083bb3f..c4813ea272 100644 --- a/dvc/scm/base.py +++ b/dvc/scm/base.py @@ -41,6 +41,12 @@ class MergeConflictError(SCMError): pass +class InvalidRemoteSCMRepo(SCMError): + def __init__(self, url: str): + msg = f"'{url}' is not a valid Git remote or URL" + super().__init__(msg) + + class Base: """Base class for source control management driver implementations.""" diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 2b37e2a8e3..2370ee5fc9 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -20,7 +20,7 @@ from dvc.path_info import PathInfo from dvc.progress import Tqdm -from dvc.scm.base import SCMError +from dvc.scm.base import InvalidRemoteSCMRepo, SCMError from dvc.utils import relpath from ..objects import GitObject @@ -355,9 +355,7 @@ def iter_remote_refs(self, url: str, base: Optional[str] = None): _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location) except Exception as exc: - raise SCMError( - f"'{url}' is not a valid Git remote or URL" - ) from exc + raise InvalidRemoteSCMRepo(url) from exc try: if base: @@ -369,7 +367,7 @@ def iter_remote_refs(self, url: str, base: Optional[str] = None): else: yield from (os.fsdecode(ref) for ref in client.get_refs(path)) except NotGitRepository as exc: - raise SCMError(f"{url} is not a git repository") from exc + raise InvalidRemoteSCMRepo(url) from exc def get_refs_containing(self, rev: str, pattern: Optional[str] = None): raise NotImplementedError @@ -655,8 +653,6 @@ def validate_git_repo(self, url: str): _, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location) except Exception as exc: - raise SCMError( - f"'{url}' is not a valid Git remote or URL" - ) from exc + raise InvalidRemoteSCMRepo(url) from exc if isinstance(client, LocalGitClient) and not os.path.exists(path): - raise SCMError(f"'{url}' is not a valid Git remote or URL") + raise InvalidRemoteSCMRepo(url) diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index 101da1a9f7..f9083fff0b 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -9,7 +9,7 @@ from dvc.repo.experiments import MultipleBranchError from dvc.repo.experiments.base import EXEC_APPLY, EXEC_CHECKPOINT from dvc.repo.experiments.utils import exp_refs_by_rev -from dvc.scm.base import SCMError +from dvc.scm.base import InvalidRemoteSCMRepo @pytest.mark.parametrize("workspace", [True, False]) @@ -256,7 +256,7 @@ def test_auto_push_error_url( ): monkeypatch.setenv(DVC_EXP_GIT_REMOTE, "true") monkeypatch.setenv(DVC_EXP_AUTO_PUSH, "true") - with pytest.raises(SCMError): + with pytest.raises(InvalidRemoteSCMRepo): dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) From 060ec01579fc8ae2be45a9a8ac814e090949df15 Mon Sep 17 00:00:00 2001 From: Gao Date: Tue, 10 Aug 2021 16:45:41 +0800 Subject: [PATCH 19/24] Update dvc/repo/experiments/executor/base.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Peter Rowlands (변기호) --- dvc/repo/experiments/executor/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 54c1402246..f685fd1854 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -489,10 +489,10 @@ def _auto_push( push_cache=push_cache, run_cache=run_cache, ) - except BaseException: + except BaseException as exc: logger.warning( - "Something went wrong while auto pushing checkpoint cache " - f"to the remote {git_remote}" + "Something went wrong while auto pushing experiment " + f"to the remote '{git_remote}': {exc}" ) @classmethod From ffb657acbc160f2e2f724ee5c131b2598d578de1 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Tue, 10 Aug 2021 17:22:58 +0800 Subject: [PATCH 20/24] Use spy tests call counts and args --- tests/func/experiments/test_checkpoints.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index f9083fff0b..c8c796a129 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -8,6 +8,7 @@ from dvc.exceptions import DvcException from dvc.repo.experiments import MultipleBranchError from dvc.repo.experiments.base import EXEC_APPLY, EXEC_CHECKPOINT +from dvc.repo.experiments.executor.base import BaseExecutor from dvc.repo.experiments.utils import exp_refs_by_rev from dvc.scm.base import InvalidRemoteSCMRepo @@ -213,18 +214,18 @@ def test_auto_push_during_iterations( local_remote, use_url, monkeypatch, + mocker, clear_env, ): # set up remote repo remote = git_upstream.url if use_url else git_upstream.remote git_upstream.scm.fetch_refspecs(str(tmp_dir), ["master:master"]) monkeypatch.setenv(DVC_EXP_GIT_REMOTE, remote) + auto_push_spy = mocker.spy(BaseExecutor, "_auto_push") # without auto push results = dvc.experiments.run(checkpoint_stage.addressing) - exp = first(results) - ref_info = first(exp_refs_by_rev(scm, exp)) - assert git_upstream.scm.get_ref(str(ref_info)) is None + assert auto_push_spy.call_count == 0 # add auto push monkeypatch.setenv(DVC_EXP_AUTO_PUSH, "true") @@ -234,6 +235,9 @@ def test_auto_push_during_iterations( ref_info = first(exp_refs_by_rev(scm, exp)) assert git_upstream.scm.get_ref(str(ref_info)) == exp + assert auto_push_spy.call_count == 2 + assert auto_push_spy.call_args[0][2] == remote + # check the data with git_upstream.dvc.config.edit() as conf: conf["remote"]["local"] = local_remote.config From 41aeaaf9e26d102a68593f32a30f91e0fce7849c Mon Sep 17 00:00:00 2001 From: Gao Date: Wed, 11 Aug 2021 14:02:38 +0800 Subject: [PATCH 21/24] Update dvc/scm/git/backend/base.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Peter Rowlands (변기호) --- dvc/scm/git/backend/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/scm/git/backend/base.py b/dvc/scm/git/backend/base.py index bb11390e8d..ad950574a0 100644 --- a/dvc/scm/git/backend/base.py +++ b/dvc/scm/git/backend/base.py @@ -349,5 +349,5 @@ def merge( """ @abstractmethod - def validate_git_repo(self, url: str): - """validate remote git repo""" + def validate_git_remote(self, url: str): + """Verify that url is a valid git URL or remote name.""" From 488c7f36d1e824932092ea4208e4d67217c14dda Mon Sep 17 00:00:00 2001 From: Gao Date: Wed, 11 Aug 2021 14:03:01 +0800 Subject: [PATCH 22/24] Update dvc/scm/git/backend/dulwich.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Peter Rowlands (변기호) --- dvc/scm/git/backend/dulwich.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 2370ee5fc9..05a847141b 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -654,5 +654,5 @@ def validate_git_repo(self, url: str): client, path = get_transport_and_path(location) except Exception as exc: raise InvalidRemoteSCMRepo(url) from exc - if isinstance(client, LocalGitClient) and not os.path.exists(path): + if isinstance(client, LocalGitClient) and not os.path.exists(os.path.join("", path)): raise InvalidRemoteSCMRepo(url) From 4a1622670f79a9db774c93a77adb5ce14db49581 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 06:04:16 +0000 Subject: [PATCH 23/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dvc/scm/git/backend/dulwich.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 05a847141b..6842f78033 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -654,5 +654,7 @@ def validate_git_repo(self, url: str): client, path = get_transport_and_path(location) except Exception as exc: raise InvalidRemoteSCMRepo(url) from exc - if isinstance(client, LocalGitClient) and not os.path.exists(os.path.join("", path)): + if isinstance(client, LocalGitClient) and not os.path.exists( + os.path.join("", path) + ): raise InvalidRemoteSCMRepo(url) From 32677a230a6a925188f77f3a1964b6fc47d57f68 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Wed, 11 Aug 2021 14:08:39 +0800 Subject: [PATCH 24/24] Remove validate_git_repo to validate_git_remote --- dvc/repo/experiments/executor/base.py | 2 +- dvc/scm/git/__init__.py | 2 +- dvc/scm/git/backend/dulwich.py | 2 +- dvc/scm/git/backend/gitpython.py | 2 +- dvc/scm/git/backend/pygit2.py | 2 +- tests/func/experiments/test_checkpoints.py | 16 ---------------- 6 files changed, 5 insertions(+), 21 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index f685fd1854..729836f152 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -260,7 +260,7 @@ def _validate_remotes(cls, dvc: "Repo", git_remote: Optional[str]): "will automatically be pushed to the default DVC remote " "(if any) on each experiment commit." ) - dvc.scm.validate_git_repo(git_remote) + dvc.scm.validate_git_remote(git_remote) dvc.cloud.get_remote_odb() @classmethod diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 629038228f..41eef55a63 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -392,7 +392,7 @@ def get_fs(self, rev: str): checkout_index = partialmethod(_backend_func, "checkout_index") status = partialmethod(_backend_func, "status") merge = partialmethod(_backend_func, "merge") - validate_git_repo = partialmethod(_backend_func, "validate_git_repo") + validate_git_remote = partialmethod(_backend_func, "validate_git_remote") def resolve_rev(self, rev: str) -> str: from dvc.repo.experiments.utils import exp_refs_by_name diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 6842f78033..12e1691426 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -645,7 +645,7 @@ def merge( ) -> Optional[str]: raise NotImplementedError - def validate_git_repo(self, url: str): + def validate_git_remote(self, url: str): from dulwich.client import LocalGitClient, get_transport_and_path from dulwich.porcelain import get_remote_repo diff --git a/dvc/scm/git/backend/gitpython.py b/dvc/scm/git/backend/gitpython.py index 3ba57ced7a..4c9b40896f 100644 --- a/dvc/scm/git/backend/gitpython.py +++ b/dvc/scm/git/backend/gitpython.py @@ -619,5 +619,5 @@ def merge( raise SCMError("Merge failed") from exc return None - def validate_git_repo(self, url: str): + def validate_git_remote(self, url: str): raise NotImplementedError diff --git a/dvc/scm/git/backend/pygit2.py b/dvc/scm/git/backend/pygit2.py index 6c3603b3c6..db30f441db 100644 --- a/dvc/scm/git/backend/pygit2.py +++ b/dvc/scm/git/backend/pygit2.py @@ -562,5 +562,5 @@ def merge( self.repo.index.write() return None - def validate_git_repo(self, url: str): + def validate_git_remote(self, url: str): raise NotImplementedError diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index c8c796a129..4859b462ca 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -238,22 +238,6 @@ def test_auto_push_during_iterations( assert auto_push_spy.call_count == 2 assert auto_push_spy.call_args[0][2] == remote - # check the data - with git_upstream.dvc.config.edit() as conf: - conf["remote"]["local"] = local_remote.config - conf["core"]["remote"] = "local" - - git_upstream.dvc.experiments.apply(ref_info.name) - git_upstream.dvc.experiments.apply(exp) - git_upstream.dvc.pull() - assert (git_upstream / "foo").read_text() == "4" - - # resume the remote checkpoint - monkeypatch.delenv(DVC_EXP_AUTO_PUSH, raising=False) - with git_upstream.chdir(): - git_upstream.dvc.experiments.run(checkpoint_stage.addressing) - assert (git_upstream / "foo").read_text() == "6" - def test_auto_push_error_url( dvc, scm, checkpoint_stage, local_remote, monkeypatch, clear_env