From 6ae33e9cbac04b73332287b82355683233e22629 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 4 May 2020 19:12:00 +0300 Subject: [PATCH] run: use multistage dvcfiles by default Adding hidden `--single-stage` to simplify the test migration. --- dvc/command/run.py | 11 +- dvc/repo/run.py | 14 ++- tests/func/test_add.py | 6 +- tests/func/test_api.py | 1 + tests/func/test_checkout.py | 16 ++- tests/func/test_commit.py | 2 +- tests/func/test_data_cloud.py | 10 +- tests/func/test_dvcfile.py | 26 +++- tests/func/test_pipeline.py | 39 ++++-- tests/func/test_repro.py | 99 ++++++++++++--- tests/func/test_repro_multistage.py | 23 +++- tests/func/test_run_multistage.py | 6 +- tests/func/test_run_single_stage.py | 179 ++++++++++++++++++++++------ 13 files changed, 345 insertions(+), 87 deletions(-) diff --git a/dvc/command/run.py b/dvc/command/run.py index ac6b1b5e93..67ff219e46 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -50,6 +50,7 @@ def run(self): outs_persist_no_cache=self.args.outs_persist_no_cache, always_changed=self.args.always_changed, name=self.args.name, + single_stage=self.args.single_stage, ) except DvcException: logger.exception("") @@ -96,7 +97,9 @@ def add_parser(subparsers, parent_parser): help="Declare dependencies for reproducible cmd.", metavar="", ) - run_parser.add_argument("-n", "--name", help=argparse.SUPPRESS) + run_parser.add_argument( + "-n", "--name", help="Stage name.", + ) run_parser.add_argument( "-o", "--outs", @@ -198,6 +201,12 @@ def add_parser(subparsers, parent_parser): default=False, help="Always consider this DVC-file as changed.", ) + run_parser.add_argument( + "--single-stage", + action="store_true", + default=False, + help=argparse.SUPPRESS, + ) run_parser.add_argument( "command", nargs=argparse.REMAINDER, help="Command to execute." ) diff --git a/dvc/repo/run.py b/dvc/repo/run.py index 65bef89d76..146f9186c7 100644 --- a/dvc/repo/run.py +++ b/dvc/repo/run.py @@ -2,6 +2,7 @@ from . import locked from .scm_context import scm_context +from dvc.exceptions import InvalidArgumentError from dvc.stage.exceptions import DuplicateStageName, InvalidStageName from funcy import first, concat @@ -36,14 +37,23 @@ def _get_file_path(kwargs): @locked @scm_context -def run(self, fname=None, no_exec=False, **kwargs): +def run(self, fname=None, no_exec=False, single_stage=False, **kwargs): from dvc.stage import PipelineStage, Stage, create_stage from dvc.dvcfile import Dvcfile, PIPELINE_FILE stage_cls = PipelineStage path = PIPELINE_FILE stage_name = kwargs.get("name") - if not stage_name: + + if stage_name and single_stage: + raise InvalidArgumentError( + "`-n|--name` is incompatible with `--single-stage`" + ) + + if not stage_name and not single_stage: + raise InvalidArgumentError("`-n|--name` is required") + + if single_stage: kwargs.pop("name", None) stage_cls = Stage path = fname or _get_file_path(kwargs) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 2c6ee387c6..79b58a2eaf 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -264,7 +264,7 @@ def test_should_update_state_entry_for_file_after_add(mocker, dvc, tmp_dir): assert ret == 0 assert file_md5_counter.mock.call_count == 1 - ret = main(["run", "-d", "foo", "echo foo"]) + ret = main(["run", "--single-stage", "-d", "foo", "echo foo"]) assert ret == 0 assert file_md5_counter.mock.call_count == 1 @@ -297,7 +297,9 @@ def test_should_update_state_entry_for_directory_after_add( assert file_md5_counter.mock.call_count == 3 ls = "dir" if os.name == "nt" else "ls" - ret = main(["run", "-d", "data", "{} {}".format(ls, "data")]) + ret = main( + ["run", "--single-stage", "-d", "data", "{} {}".format(ls, "data")] + ) assert ret == 0 assert file_md5_counter.mock.call_count == 3 diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 00b0500cac..943312fe07 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -129,6 +129,7 @@ def test_open_not_cached(dvc): metric_file, metric_content ) dvc.run( + single_stage=True, metrics_no_cache=[metric_file], cmd=('python -c "{}"'.format(metric_code)), ) diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 56c58f23d4..0c8f97354b 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -256,6 +256,7 @@ def test(self): self.commit_data_file(fname1) self.commit_data_file(fname2) self.dvc.run( + single_stage=True, cmd="python {} {} {}".format(self.CODE, self.FOO, fname3), deps=[self.CODE, self.FOO], outs_no_cache=[fname3], @@ -341,7 +342,10 @@ def test(self): self.dvc.add(self.FOO) stage = self.dvc.run( - cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"] + cmd=cmd, + deps=[self.FOO, self.CODE], + outs_no_cache=["out"], + single_stage=True, ) self.assertTrue(stage is not None) @@ -480,7 +484,9 @@ def test(self): def test_checkout_no_checksum(tmp_dir, dvc): tmp_dir.gen("file", "file content") - stage = dvc.run(outs=["file"], no_exec=True, cmd="somecmd") + stage = dvc.run( + outs=["file"], no_exec=True, cmd="somecmd", single_stage=True + ) with pytest.raises(CheckoutError): dvc.checkout([stage.path], force=True) @@ -706,7 +712,11 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link): def test_checkout_with_deps(tmp_dir, dvc): tmp_dir.dvc_gen({"foo": "foo"}) dvc.run( - fname="copy_file.dvc", cmd="echo foo > bar", outs=["bar"], deps=["foo"] + fname="copy_file.dvc", + cmd="echo foo > bar", + outs=["bar"], + deps=["foo"], + single_stage=True, ) (tmp_dir / "bar").unlink() diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index 51171caf01..951c8d4ef7 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -38,7 +38,7 @@ def test_commit_force(tmp_dir, dvc): assert dvc.status([stage.path]) == {} -@pytest.mark.parametrize("run_kw", [{}, {"name": "copy"}]) +@pytest.mark.parametrize("run_kw", [{"single_stage": True}, {"name": "copy"}]) def test_commit_with_deps(tmp_dir, dvc, run_copy, run_kw): tmp_dir.gen("foo", "foo") (foo_stage,) = dvc.add("foo", no_commit=True) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index a9336edee3..00459ffc15 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -520,7 +520,9 @@ def _test(self): url = Local.get_url() self.main(["remote", "add", "-d", TEST_REMOTE, url]) - stage = self.dvc.run(outs=["bar"], cmd="echo bar > bar") + stage = self.dvc.run( + outs=["bar"], cmd="echo bar > bar", single_stage=True + ) self.main(["push"]) stage_file_path = stage.relpath @@ -630,7 +632,7 @@ def test_checksum_recalculation(mocker, dvc, tmp_dir): assert ret == 0 ret = main(["push"]) assert ret == 0 - ret = main(["run", "-d", "foo", "echo foo"]) + ret = main(["run", "--single-stage", "-d", "foo", "echo foo"]) assert ret == 0 assert test_get_file_checksum.mock.call_count == 1 @@ -783,7 +785,7 @@ def recurse_list_dir(d): def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy): (stage0,) = tmp_dir.dvc_gen("foo", "foo") - stage1 = run_copy("foo", "bar") + stage1 = run_copy("foo", "bar", single_stage=True) stage2 = run_copy("bar", "foobar", name="copy-bar-foobar") outs = ["foo", "bar", "foobar"] @@ -813,7 +815,7 @@ def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy): def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy): tmp_dir.dvc_gen("foo", "foo") - run_copy("foo", "bar") + run_copy("foo", "bar", single_stage=True) tmp_dir.dvc_gen("lorem", "lorem") run_copy("lorem", "lorem2", name="copy-lorem-lorem2") diff --git a/tests/func/test_dvcfile.py b/tests/func/test_dvcfile.py index faee4065b1..3727b1d66c 100644 --- a/tests/func/test_dvcfile.py +++ b/tests/func/test_dvcfile.py @@ -50,7 +50,11 @@ def test_run_load_one_for_multistage_non_existing_stage_name(tmp_dir, dvc): def test_run_load_one_on_single_stage(tmp_dir, dvc): tmp_dir.gen("foo", "foo") stage = dvc.run( - cmd="cp foo foo2", deps=["foo"], metrics=["foo2"], always_changed=True, + cmd="cp foo foo2", + deps=["foo"], + metrics=["foo2"], + always_changed=True, + single_stage=True, ) assert Dvcfile(dvc, stage.path).stages.get("random-name") assert Dvcfile(dvc, stage.path).stage @@ -97,7 +101,11 @@ def test_load_all_multistage(tmp_dir, dvc): def test_load_all_singlestage(tmp_dir, dvc): tmp_dir.gen("foo", "foo") stage1 = dvc.run( - cmd="cp foo foo2", deps=["foo"], metrics=["foo2"], always_changed=True, + cmd="cp foo foo2", + deps=["foo"], + metrics=["foo2"], + always_changed=True, + single_stage=True, ) stages = Dvcfile(dvc, "foo2.dvc").stages.values() assert len(stages) == 1 @@ -107,7 +115,11 @@ def test_load_all_singlestage(tmp_dir, dvc): def test_load_singlestage(tmp_dir, dvc): tmp_dir.gen("foo", "foo") stage1 = dvc.run( - cmd="cp foo foo2", deps=["foo"], metrics=["foo2"], always_changed=True, + cmd="cp foo foo2", + deps=["foo"], + metrics=["foo2"], + always_changed=True, + single_stage=True, ) assert Dvcfile(dvc, "foo2.dvc").stage == stage1 @@ -144,7 +156,11 @@ def test_stage_collection(tmp_dir, dvc): always_changed=True, ) stage3 = dvc.run( - cmd="cp bar bar2", deps=["bar"], metrics=["bar2"], always_changed=True, + cmd="cp bar bar2", + deps=["bar"], + metrics=["bar2"], + always_changed=True, + single_stage=True, ) assert {s for s in dvc.stages} == {stage1, stage3, stage2} @@ -174,7 +190,7 @@ def test_stage_filter(tmp_dir, dvc, run_copy): def test_stage_filter_in_singlestage_file(tmp_dir, dvc, run_copy): tmp_dir.gen("foo", "foo") - stage = run_copy("foo", "bar") + stage = run_copy("foo", "bar", single_stage=True) dvcfile = Dvcfile(dvc, stage.path) assert set(dvcfile.stages.filter(None).values()) == {stage} assert dvcfile.stages.filter(None).get(None) == stage diff --git a/tests/func/test_pipeline.py b/tests/func/test_pipeline.py index 6da16f7b7b..eda4a14713 100644 --- a/tests/func/test_pipeline.py +++ b/tests/func/test_pipeline.py @@ -103,10 +103,23 @@ def test_disconnected_stage(tmp_dir, dvc): tmp_dir.dvc_gen({"base": "base"}) dvc.add("base") - dvc.run(deps=["base"], outs=["derived1"], cmd="echo derived1 > derived1") - dvc.run(deps=["base"], outs=["derived2"], cmd="echo derived2 > derived2") + dvc.run( + deps=["base"], + outs=["derived1"], + cmd="echo derived1 > derived1", + single_stage=True, + ) + dvc.run( + deps=["base"], + outs=["derived2"], + cmd="echo derived2 > derived2", + single_stage=True, + ) final_stage = dvc.run( - deps=["derived1"], outs=["final"], cmd="echo final > final" + deps=["derived1"], + outs=["final"], + cmd="echo final > final", + single_stage=True, ) command = CmdPipelineShow([]) @@ -134,7 +147,7 @@ def test_print_locked_stages(tmp_dir, dvc, caplog): def test_dot_outs(tmp_dir, dvc, run_copy): tmp_dir.gen("foo", "foo content") - run_copy("foo", "file") + run_copy("foo", "file", single_stage=True) assert main(["pipeline", "show", "--dot", "file.dvc", "--outs"]) == 0 @@ -208,8 +221,10 @@ def test_no_stages(self): def one_pipeline(self): self.dvc.add("foo") - self.dvc.run(deps=["foo"], outs=["bar"], cmd="") - self.dvc.run(deps=["bar"], outs=["baz"], cmd="echo baz > baz") + self.dvc.run(deps=["foo"], outs=["bar"], cmd="", single_stage=True) + self.dvc.run( + deps=["bar"], outs=["baz"], cmd="echo baz > baz", single_stage=True + ) pipelines = self.dvc.pipelines self.assertEqual(len(pipelines), 1) @@ -218,8 +233,10 @@ def one_pipeline(self): def two_pipelines(self): self.dvc.add("foo") - self.dvc.run(deps=["foo"], outs=["bar"], cmd="") - self.dvc.run(deps=["bar"], outs=["baz"], cmd="echo baz > baz") + self.dvc.run(deps=["foo"], outs=["bar"], cmd="", single_stage=True) + self.dvc.run( + deps=["bar"], outs=["baz"], cmd="echo baz > baz", single_stage=True + ) self.dvc.add("code.py") @@ -248,11 +265,13 @@ def test_split_pipeline(tmp_dir, scm, dvc): deps=["git_dep1", "data"], outs=["data_train", "data_valid"], cmd="echo train >> data_train && echo valid >> data_valid", + single_stage=True, ) stage = dvc.run( deps=["git_dep2", "data_train", "data_valid"], outs=["result"], cmd="echo result >> result", + single_stage=True, ) command = CmdPipelineShow([]) @@ -271,7 +290,7 @@ def test_split_pipeline(tmp_dir, scm, dvc): def test_pipeline_list_show_multistage(tmp_dir, dvc, run_copy, caplog): tmp_dir.gen("foo", "foo") run_copy("foo", "bar", name="copy-foo-bar") - run_copy("bar", "foobar") + run_copy("bar", "foobar", single_stage=True) command = CmdPipelineShow([]) caplog.clear() @@ -299,7 +318,7 @@ def test_pipeline_list_show_multistage(tmp_dir, dvc, run_copy, caplog): def test_pipeline_ascii_multistage(tmp_dir, dvc, run_copy): tmp_dir.gen("foo", "foo") run_copy("foo", "bar", name="copy-foo-bar") - run_copy("bar", "foobar") + run_copy("bar", "foobar", single_stage=True) command = CmdPipelineShow([]) nodes, edges, is_tree = command._build_graph("foobar.dvc") assert set(nodes) == {"dvc.yaml:copy-foo-bar", "foobar.dvc"} diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index b7ce68eafb..c0ba14c412 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -52,6 +52,7 @@ class SingleStageRun: def _run(self, **kwargs): + kwargs["single_stage"] = True kwargs.pop("name", None) return self.dvc.run(**kwargs) @@ -140,6 +141,7 @@ def test(self): wdir="dir1", outs=[os.path.join("..", "dir2")], cmd="mkdir {path}".format(path=os.path.join("..", "dir2")), + single_stage=True, ) faulty_stage_path = os.path.join("dir2", "something.dvc") @@ -176,6 +178,7 @@ def test_nested(self): wdir=dir1, outs=[out_dir], # ../a/nested cmd="mkdir {path}".format(path=out_dir), + single_stage=True, ) os.mkdir(os.path.join(nested_dir, "dir")) @@ -210,7 +213,9 @@ def test_similar_paths(self): # |-- a # |__ a.dvc (stage.cwd == something-1) - self.dvc.run(outs=["something"], cmd="mkdir something") + self.dvc.run( + outs=["something"], cmd="mkdir something", single_stage=True + ) os.mkdir("something-1") @@ -262,7 +267,9 @@ def test(self): self.assertTrue(stages[0] is not None) stage = self.dvc.run( - fname="dvcfile2.dvc", deps=[self.DATA, self.DATA_SUB] + fname="dvcfile2.dvc", + deps=[self.DATA, self.DATA_SUB], + single_stage=True, ) self.assertTrue(stage is not None) @@ -388,6 +395,7 @@ def test(self): [ "run", "--no-exec", + "--single-stage", "-d", idir, "-o", @@ -398,7 +406,9 @@ def test(self): ) self.assertEqual(ret, 0) - ret = main(["run", "--no-exec", "-f", "Dvcfile"] + deps) + ret = main( + ["run", "--no-exec", "--single-stage", "-f", "Dvcfile"] + deps + ) self.assertEqual(ret, 0) ret = main(["repro", "--dry"]) @@ -442,6 +452,7 @@ def test(self): outs=[file1], deps=[self.FOO, code1], cmd="python {} {} {}".format(code1, self.FOO, file1), + single_stage=True, ) self.assertTrue(file1_stage is not None) @@ -452,6 +463,7 @@ def test(self): outs=[file2], deps=[file1, code2], cmd="python {} {} {}".format(code2, file1, file2), + single_stage=True, ) self.assertTrue(file2_stage is not None) @@ -462,6 +474,7 @@ def test(self): outs=[file3], deps=[file2, code3], cmd="python {} {} {}".format(code3, file2, file3), + single_stage=True, ) self.assertTrue(file3_stage is not None) @@ -508,6 +521,7 @@ def setUp(self): outs=[self.file1], deps=[self.FOO, self.CODE], cmd="python {} {} {}".format(self.CODE, self.FOO, self.file1), + single_stage=True, ) self.file2 = "file2" @@ -626,6 +640,7 @@ def test(self): outs_no_cache=[file1], deps=[self.FOO, self.CODE], cmd="python {} {} {}".format(self.CODE, self.FOO, file1), + single_stage=True, ) stages = self.dvc.reproduce(file1_stage) @@ -786,6 +801,7 @@ def test(self): ret = main( [ "run", + "--single-stage", "-f", "{}/Dvcfile".format(dname), "-w", @@ -1231,7 +1247,10 @@ def test(self): stage = fname + ".dvc" self.dvc.run( - fname=stage, outs=[fname], cmd="echo $SHELL > {}".format(fname) + fname=stage, + outs=[fname], + cmd="echo $SHELL > {}".format(fname), + single_stage=True, ) with open(fname, "r") as fd: @@ -1320,6 +1339,7 @@ def test_force_with_dependencies(self): deps=[self.FOO], outs=["datetime.txt"], cmd='python -c "import time; print(time.time())" > datetime.txt', + single_stage=True, ).outs[0] ret = main(["repro", "--force", "datetime.dvc"]) @@ -1363,6 +1383,7 @@ def test(self): ret = main( [ "run", + "--single-stage", "-m", metrics_file, "echo {} >> {}".format(metrics_value, metrics_file), @@ -1409,7 +1430,7 @@ def repro_dir(tmp_dir, dvc, run_copy): stages = {} origin_copy = tmp_dir / "origin_copy" - stage = run_copy("origin_data", fspath(origin_copy)) + stage = run_copy("origin_data", fspath(origin_copy), single_stage=True) assert stage is not None assert origin_copy.read_text() == "origin data content" stages["origin_copy"] = stage @@ -1419,6 +1440,7 @@ def repro_dir(tmp_dir, dvc, run_copy): fspath(origin_copy), fspath(origin_copy_2), fname=fspath(origin_copy_2) + ".dvc", + single_stage=True, ) assert stage is not None assert origin_copy_2.read_text() == "origin data content" @@ -1430,6 +1452,7 @@ def repro_dir(tmp_dir, dvc, run_copy): fspath(dir_file_path), fspath(dir_file_copy), fname=fspath(dir_file_copy) + ".dvc", + single_stage=True, ) assert stage is not None assert dir_file_copy.read_text() == "dir file content" @@ -1439,13 +1462,20 @@ def repro_dir(tmp_dir, dvc, run_copy): stage = dvc.run( fname=fspath(last_stage), deps=[fspath(origin_copy_2), fspath(dir_file_copy)], + single_stage=True, ) assert stage is not None stages["last_stage"] = stage # Unrelated are to verify that reproducing `dir` will not trigger them too - assert run_copy(fspath(origin_copy), "unrelated1") is not None - assert run_copy(fspath(dir_file_path), "unrelated2") is not None + assert ( + run_copy(fspath(origin_copy), "unrelated1", single_stage=True) + is not None + ) + assert ( + run_copy(fspath(dir_file_path), "unrelated2", single_stage=True) + is not None + ) yield stages @@ -1554,7 +1584,9 @@ def test_recursive_repro_on_stage_file(dvc, repro_dir): def test_dvc_formatting_retained(tmp_dir, dvc, run_copy): tmp_dir.dvc_gen("foo", "foo content") - stage = run_copy("foo", "foo_copy", fname="foo_copy.dvc") + stage = run_copy( + "foo", "foo_copy", fname="foo_copy.dvc", single_stage=True + ) stage_path = tmp_dir / stage.relpath # Add comments and custom formatting to DVC-file @@ -1597,13 +1629,49 @@ def test_downstream(dvc): # \ / # A # - assert main(["run", "-o", "A", "echo A>A"]) == 0 - assert main(["run", "-d", "A", "-o", "B", "echo B>B"]) == 0 - assert main(["run", "-d", "A", "-o", "C", "echo C>C"]) == 0 - assert main(["run", "-d", "B", "-d", "C", "-o", "D", "echo D>D"]) == 0 - assert main(["run", "-o", "G", "echo G>G"]) == 0 - assert main(["run", "-d", "G", "-o", "F", "echo F>F"]) == 0 - assert main(["run", "-d", "D", "-d", "F", "-o", "E", "echo E>E"]) == 0 + assert main(["run", "--single-stage", "-o", "A", "echo A>A"]) == 0 + assert ( + main(["run", "--single-stage", "-d", "A", "-o", "B", "echo B>B"]) == 0 + ) + assert ( + main(["run", "--single-stage", "-d", "A", "-o", "C", "echo C>C"]) == 0 + ) + assert ( + main( + [ + "run", + "--single-stage", + "-d", + "B", + "-d", + "C", + "-o", + "D", + "echo D>D", + ] + ) + == 0 + ) + assert main(["run", "--single-stage", "-o", "G", "echo G>G"]) == 0 + assert ( + main(["run", "--single-stage", "-d", "G", "-o", "F", "echo F>F"]) == 0 + ) + assert ( + main( + [ + "run", + "--single-stage", + "-d", + "D", + "-d", + "F", + "-o", + "E", + "echo E>E", + ] + ) + == 0 + ) # We want the evaluation to move from B to E # @@ -1670,6 +1738,7 @@ def test_ssh_dir_out(tmp_dir, dvc, ssh_server): url_info = URLInfo(remote_url) repo.run( cmd="python {} {}".format(tmp_dir / "script.py", url_info.path), + single_stage=True, outs=["remote://upstream/dir-out"], deps=["foo"], # add a fake dep to not consider this a callback ) diff --git a/tests/func/test_repro_multistage.py b/tests/func/test_repro_multistage.py index 10b6ca4135..036f778bb6 100644 --- a/tests/func/test_repro_multistage.py +++ b/tests/func/test_repro_multistage.py @@ -223,16 +223,33 @@ def test_downstream(tmp_dir, dvc): # assert main(["run", "-n", "A-gen", "-o", "A", "echo A>A"]) == 0 assert main(["run", "-n", "B-gen", "-d", "A", "-o", "B", "echo B>B"]) == 0 - assert main(["run", "-d", "A", "-o", "C", "echo C>C"]) == 0 + assert ( + main(["run", "--single-stage", "-d", "A", "-o", "C", "echo C>C"]) == 0 + ) assert ( main( ["run", "-n", "D-gen", "-d", "B", "-d", "C", "-o", "D", "echo D>D"] ) == 0 ) - assert main(["run", "-o", "G", "echo G>G"]) == 0 + assert main(["run", "--single-stage", "-o", "G", "echo G>G"]) == 0 assert main(["run", "-n", "F-gen", "-d", "G", "-o", "F", "echo F>F"]) == 0 - assert main(["run", "-d", "D", "-d", "F", "-o", "E", "echo E>E"]) == 0 + assert ( + main( + [ + "run", + "--single-stage", + "-d", + "D", + "-d", + "F", + "-o", + "E", + "echo E>E", + ] + ) + == 0 + ) # We want the evaluation to move from B to E # diff --git a/tests/func/test_run_multistage.py b/tests/func/test_run_multistage.py index a156ad55ff..319a1f62ec 100644 --- a/tests/func/test_run_multistage.py +++ b/tests/func/test_run_multistage.py @@ -23,9 +23,9 @@ def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy): from dvc.stage import PipelineStage, Stage tmp_dir.dvc_gen("foo", "foo") - stage1 = run_copy("foo", "foo1") + stage1 = run_copy("foo", "foo1", single_stage=True) stage2 = run_copy("foo1", "foo2", name="copy-foo1-foo2") - stage3 = run_copy("foo2", "foo3") + stage3 = run_copy("foo2", "foo3", single_stage=True) assert isinstance(stage2, PipelineStage) assert isinstance(stage1, Stage) @@ -40,7 +40,7 @@ def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy): tmp_dir.dvc_gen("foo", "foo") run_copy("foo", "foo1", name="copy-foo-foo1") run_copy("foo1", "foo2", name="copy-foo1-foo2") - run_copy("foo2", "foo3") + run_copy("foo2", "foo3", single_stage=True) stages = list(Dvcfile(dvc, PIPELINE_FILE).stages.values()) assert len(stages) == 2 diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 1270439db7..f5b02cddb0 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -48,6 +48,7 @@ def test(self): outs=outs, outs_no_cache=outs_no_cache, fname=fname, + single_stage=True, ) self.assertTrue(filecmp.cmp(self.FOO, "out", shallow=False)) @@ -66,13 +67,19 @@ def test(self): outs=outs, outs_no_cache=outs_no_cache, fname="duplicate" + fname, + single_stage=True, ) class TestRunEmpty(TestDvc): def test(self): self.dvc.run( - cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty.dvc", + cmd="", + deps=[], + outs=[], + outs_no_cache=[], + fname="empty.dvc", + single_stage=True, ) @@ -87,6 +94,7 @@ def test(self): outs=[], outs_no_cache=[], fname="empty.dvc", + single_stage=True, ) @@ -97,6 +105,7 @@ def test(self): deps=[self.CODE, self.FOO], outs=["out"], no_exec=True, + single_stage=True, ) self.assertFalse(os.path.exists("out")) with open(".gitignore", "r") as fobj: @@ -111,6 +120,7 @@ def test(self): deps=[self.FOO], outs=[self.FOO], fname="circular-dependency.dvc", + single_stage=True, ) def test_outs_no_cache(self): @@ -120,6 +130,7 @@ def test_outs_no_cache(self): deps=[self.FOO], outs_no_cache=[self.FOO], fname="circular-dependency.dvc", + single_stage=True, ) def test_non_normalized_paths(self): @@ -129,20 +140,30 @@ def test_non_normalized_paths(self): deps=["./foo"], outs=["foo"], fname="circular-dependency.dvc", + single_stage=True, ) def test_graph(self): self.dvc.run( - deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt" + deps=[self.FOO], + outs=["bar.txt"], + cmd="echo bar > bar.txt", + single_stage=True, ) self.dvc.run( - deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt" + deps=["bar.txt"], + outs=["baz.txt"], + cmd="echo baz > baz.txt", + single_stage=True, ) with self.assertRaises(CyclicGraphError): self.dvc.run( - deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo" + deps=["baz.txt"], + outs=[self.FOO], + cmd="echo baz > foo", + single_stage=True, ) @@ -154,6 +175,7 @@ def test(self): deps=[], outs=[self.FOO, self.FOO], fname="circular-dependency.dvc", + single_stage=True, ) def test_outs_no_cache(self): @@ -163,6 +185,7 @@ def test_outs_no_cache(self): outs=[self.FOO], outs_no_cache=[self.FOO], fname="circular-dependency.dvc", + single_stage=True, ) def test_non_normalized_paths(self): @@ -172,56 +195,62 @@ def test_non_normalized_paths(self): deps=[], outs=["foo", "./foo"], fname="circular-dependency.dvc", + single_stage=True, ) class TestRunStageInsideOutput(TestDvc): def test_cwd(self): - self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR]) + self.dvc.run( + cmd="", deps=[], outs=[self.DATA_DIR], single_stage=True, + ) with self.assertRaises(StagePathAsOutputError): self.dvc.run( - cmd="", fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc") + cmd="", + fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc"), + single_stage=True, ) def test_file_name(self): - self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR]) + self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR], single_stage=True) with self.assertRaises(StagePathAsOutputError): self.dvc.run( cmd="", outs=[self.FOO], fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc"), + single_stage=True, ) class TestRunBadCwd(TestDvc): def test(self): with self.assertRaises(StagePathOutsideError): - self.dvc.run(cmd="", wdir=self.mkdtemp()) + self.dvc.run(cmd="", wdir=self.mkdtemp(), single_stage=True) def test_same_prefix(self): with self.assertRaises(StagePathOutsideError): path = "{}-{}".format(self._root_dir, uuid.uuid4()) os.mkdir(path) - self.dvc.run(cmd="", wdir=path) + self.dvc.run(cmd="", wdir=path, single_stage=True) class TestRunBadWdir(TestDvc): def test(self): with self.assertRaises(StagePathOutsideError): - self.dvc.run(cmd="", wdir=self.mkdtemp()) + self.dvc.run(cmd="", wdir=self.mkdtemp(), single_stage=True) def test_same_prefix(self): with self.assertRaises(StagePathOutsideError): path = "{}-{}".format(self._root_dir, uuid.uuid4()) os.mkdir(path) - self.dvc.run(cmd="", wdir=path) + self.dvc.run(cmd="", wdir=path, single_stage=True) def test_not_found(self): with self.assertRaises(StagePathNotFoundError): path = os.path.join(self._root_dir, str(uuid.uuid4())) - self.dvc.run(cmd="", wdir=path) + self.dvc.run(cmd="", wdir=path, single_stage=True) def test_not_dir(self): with self.assertRaises(StagePathNotDirectoryError): @@ -229,7 +258,9 @@ def test_not_dir(self): os.mkdir(path) path = os.path.join(path, str(uuid.uuid4())) open(path, "a").close() - self.dvc.run(cmd="", wdir=path) + self.dvc.run( + cmd="", wdir=path, single_stage=True, + ) class TestRunBadName(TestDvc): @@ -238,6 +269,7 @@ def test(self): self.dvc.run( cmd="", fname=os.path.join(self.mkdtemp(), self.FOO + DVC_FILE_SUFFIX), + single_stage=True, ) def test_same_prefix(self): @@ -245,14 +277,18 @@ def test_same_prefix(self): path = "{}-{}".format(self._root_dir, uuid.uuid4()) os.mkdir(path) self.dvc.run( - cmd="", fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX), + cmd="", + fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX), + single_stage=True, ) def test_not_found(self): with self.assertRaises(StagePathNotFoundError): path = os.path.join(self._root_dir, str(uuid.uuid4())) self.dvc.run( - cmd="", fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX), + cmd="", + fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX), + single_stage=True, ) @@ -269,6 +305,7 @@ def test(self): deps=[self.CODE], outs=[self.FOO], cmd="python {} {}".format(self.CODE, self.FOO), + single_stage=True, ) @@ -289,6 +326,7 @@ def test(self): self.CODE, "-o", self.FOO, + "--single-stage", "python", self.CODE, self.FOO, @@ -304,6 +342,7 @@ def test(self): "run", "--overwrite-dvcfile", "--ignore-build-cache", + "--single-stage", "-d", self.CODE, "-o", @@ -338,6 +377,7 @@ def test(self): self.CODE, "-o", self.FOO, + "--single-stage", "python", self.CODE, self.FOO, @@ -360,6 +400,7 @@ def test(self): "run", "--overwrite-dvcfile", "--ignore-build-cache", + "--single-stage", "-d", self.CODE, "-o", @@ -401,6 +442,7 @@ def test(self): self.CODE, "-o", self.FOO, + "--single-stage", "python", self.CODE, self.FOO, @@ -417,6 +459,7 @@ def test(self): "run", "--overwrite-dvcfile", "--ignore-build-cache", + "--single-stage", "-d", self.CODE, "-o", @@ -453,6 +496,7 @@ def test(self): "out", "-f", "out.dvc", + "--single-stage", "python", self.CODE, self.FOO, @@ -476,6 +520,7 @@ def test(self): "out", "-f", "out.dvc", + "--single-stage", "python", self.CODE, self.FOO, @@ -499,6 +544,7 @@ def test(self): self.CODE, "--overwrite-dvcfile", "--ignore-build-cache", + "--single-stage", "-o", "out", "-f", @@ -518,7 +564,15 @@ def test(self): time.sleep(1) ret = main( - ["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR] + [ + "run", + "--overwrite-dvcfile", + "--single-stage", + "-f", + "out.dvc", + "-d", + self.BAR, + ] ) self.assertEqual(ret, 0) @@ -528,13 +582,29 @@ def test(self): class TestCmdRunCliMetrics(TestDvc): def test_cached(self): - ret = main(["run", "-m", "metrics.txt", "echo test > metrics.txt"]) + ret = main( + [ + "run", + "-m", + "metrics.txt", + "--single-stage", + "echo test > metrics.txt", + ] + ) self.assertEqual(ret, 0) with open("metrics.txt", "r") as fd: self.assertEqual(fd.read().rstrip(), "test") def test_not_cached(self): - ret = main(["run", "-M", "metrics.txt", "echo test > metrics.txt"]) + ret = main( + [ + "run", + "-M", + "metrics.txt", + "--single-stage", + "echo test > metrics.txt", + ] + ) self.assertEqual(ret, 0) with open("metrics.txt", "r") as fd: self.assertEqual(fd.read().rstrip(), "test") @@ -543,13 +613,18 @@ def test_not_cached(self): class TestCmdRunWorkingDirectory(TestDvc): def test_default_wdir_is_not_written(self): stage = self.dvc.run( - cmd="echo test > {}".format(self.FOO), outs=[self.FOO], wdir="." + cmd="echo test > {}".format(self.FOO), + outs=[self.FOO], + wdir=".", + single_stage=True, ) d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) stage = self.dvc.run( - cmd="echo test > {}".format(self.BAR), outs=[self.BAR] + cmd="echo test > {}".format(self.BAR), + outs=[self.BAR], + single_stage=True, ) d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) @@ -560,7 +635,10 @@ def test_fname_changes_path_and_wdir(self): foo = os.path.join(dname, self.FOO) fname = os.path.join(dname, "stage" + DVC_FILE_SUFFIX) stage = self.dvc.run( - cmd="echo test > {}".format(foo), outs=[foo], fname=fname + cmd="echo test > {}".format(foo), + outs=[foo], + fname=fname, + single_stage=True, ) self.assertEqual(stage.wdir, os.path.realpath(self._root_dir)) self.assertEqual( @@ -575,21 +653,28 @@ def test_fname_changes_path_and_wdir(self): def test_rerun_deterministic(tmp_dir, run_copy): tmp_dir.gen("foo", "foo content") - assert run_copy("foo", "out") is not None - assert run_copy("foo", "out") is None + assert run_copy("foo", "out", single_stage=True) is not None + assert run_copy("foo", "out", single_stage=True) is None def test_rerun_deterministic_ignore_cache(tmp_dir, run_copy): tmp_dir.gen("foo", "foo content") - assert run_copy("foo", "out") is not None - assert run_copy("foo", "out", ignore_build_cache=True) is not None + assert run_copy("foo", "out", single_stage=True) is not None + assert ( + run_copy("foo", "out", ignore_build_cache=True, single_stage=True) + is not None + ) def test_rerun_callback(dvc): def run_callback(): return dvc.run( - cmd=("echo content > out"), outs=["out"], deps=[], overwrite=False + cmd=("echo content > out"), + outs=["out"], + deps=[], + overwrite=False, + single_stage=True, ) assert run_callback() is not None @@ -600,36 +685,46 @@ def run_callback(): def test_rerun_changed_dep(tmp_dir, run_copy): tmp_dir.gen("foo", "foo content") - assert run_copy("foo", "out") is not None + assert run_copy("foo", "out", single_stage=True) is not None tmp_dir.gen("foo", "changed content") with pytest.raises(StageFileAlreadyExistsError): - run_copy("foo", "out", overwrite=False) + run_copy("foo", "out", overwrite=False, single_stage=True) def test_rerun_changed_stage(tmp_dir, run_copy): tmp_dir.gen("foo", "foo content") - assert run_copy("foo", "out") is not None + assert run_copy("foo", "out", single_stage=True) is not None tmp_dir.gen("bar", "bar content") with pytest.raises(StageFileAlreadyExistsError): - run_copy("bar", "out", overwrite=False) + run_copy("bar", "out", overwrite=False, single_stage=True) def test_rerun_changed_out(tmp_dir, run_copy): tmp_dir.gen("foo", "foo content") - assert run_copy("foo", "out") is not None + assert run_copy("foo", "out", single_stage=True) is not None Path("out").write_text("modification") with pytest.raises(StageFileAlreadyExistsError): - run_copy("foo", "out", overwrite=False) + run_copy("foo", "out", overwrite=False, single_stage=True) class TestRunCommit(TestDvc): def test(self): fname = "test" ret = main( - ["run", "-o", fname, "--no-commit", "echo", "test", ">", fname] + [ + "run", + "-o", + fname, + "--no-commit", + "--single-stage", + "echo", + "test", + ">", + fname, + ] ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(fname)) @@ -662,6 +757,7 @@ def run_command(self, file, file_content): ret = main( [ "run", + "--single-stage", self.outs_command, file, "echo {} >> {}".format(file_content, file), @@ -715,7 +811,9 @@ def test(self): with self.assertRaises(OverlappingOutputPathsError) as err: self.dvc.run( - outs=[self.DATA], cmd="echo data >> {}".format(self.DATA) + outs=[self.DATA], + cmd="echo data >> {}".format(self.DATA), + single_stage=True, ) error_output = str(err.exception) @@ -750,6 +848,7 @@ def _run_twice_with_same_outputs(self): ret = main( [ "run", + "--single-stage", "--outs", self.FOO, "echo {} > {}".format(self.FOO_CONTENTS, self.FOO), @@ -766,6 +865,7 @@ def _run_twice_with_same_outputs(self): self._outs_command, self.FOO, "--overwrite-dvcfile", + "--single-stage", "echo {} >> {}".format(self.BAR_CONTENTS, self.FOO), ] ) @@ -807,7 +907,9 @@ def setUp(self): def test(self): cmd = "python {} {} {}".format(self.CODE, self.FOO, self.BAR) - stage = self.dvc.run(deps=[self.FOO], outs=[self.BAR], cmd=cmd) + stage = self.dvc.run( + deps=[self.FOO], outs=[self.BAR], cmd=cmd, single_stage=True + ) os.chmod(self.BAR, 0o644) with open(self.BAR, "w") as fd: @@ -837,6 +939,7 @@ def test_ignore_build_cache(self): cmd = [ "run", "--overwrite-dvcfile", + "--single-stage", "--deps", "immutable", "--outs-persist", @@ -862,7 +965,7 @@ def test_bad_stage_fname(tmp_dir, dvc, run_copy): with pytest.raises(StageFileBadNameError): # fname should end with .dvc - run_copy("foo", "foo_copy", fname="out_stage") + run_copy("foo", "foo_copy", fname="out_stage", single_stage=True) # Check that command hasn't been run assert not (tmp_dir / "foo_copy").exists() @@ -870,11 +973,11 @@ def test_bad_stage_fname(tmp_dir, dvc, run_copy): def test_should_raise_on_stage_dependency(run_copy): with pytest.raises(DependencyIsStageFileError): - run_copy("name.dvc", "stage_copy") + run_copy("name.dvc", "stage_copy", single_stage=True) def test_should_raise_on_stage_output(tmp_dir, dvc, run_copy): tmp_dir.dvc_gen("foo", "foo content") with pytest.raises(OutputIsStageFileError): - run_copy("foo", "name.dvc") + run_copy("foo", "name.dvc", single_stage=True)