Skip to content

tests: remotes: use TmpDir-like fixtures #4140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Parent: dvc.yaml: added a new option to dvc.yaml to avoid unprotecting files
merged 1 commit into from
Jul 2, 2020
Merged

tests: remotes: use TmpDir-like fixtures #4140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion setup.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ count=true
[isort]
include_trailing_comma=true
known_first_party=dvc,tests
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
line_length=79
force_grid_wrap=0
use_parentheses=True
Expand Down
106 changes: 88 additions & 18 deletions tests/func/test_add.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,27 +185,97 @@ def test_add_file_in_dir(tmp_dir, dvc):
assert stage.outs[0].def_path == "subdata"


class TestAddExternalLocalFile(TestDvc):
def test(self):
from dvc.stage.exceptions import StageExternalOutputsError
@pytest.mark.parametrize(
"workspace, hash_name, hash_value",
[
(
pytest.lazy_fixture("local_cloud"),
"md5",
"8c7dd922ad47494fc02c388e12c00eac",
),
pytest.param(
pytest.lazy_fixture("ssh"),
"md5",
"8c7dd922ad47494fc02c388e12c00eac",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
(
pytest.lazy_fixture("s3"),
"etag",
"8c7dd922ad47494fc02c388e12c00eac",
),
(pytest.lazy_fixture("gs"), "md5", "8c7dd922ad47494fc02c388e12c00eac"),
(
pytest.lazy_fixture("hdfs"),
"checksum",
"000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
),
],
indirect=["workspace"],
)
def test_add_external_file(tmp_dir, dvc, workspace, hash_name, hash_value):
from dvc.stage.exceptions import StageExternalOutputsError

dname = TestDvc.mkdtemp()
fname = os.path.join(dname, "foo")
shutil.copyfile(self.FOO, fname)
workspace.gen("file", "file")

with self.assertRaises(StageExternalOutputsError):
self.dvc.add(fname)
with pytest.raises(StageExternalOutputsError):
dvc.add(workspace.url)

stages = self.dvc.add(fname, external=True)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertNotEqual(stage, None)
self.assertEqual(len(stage.deps), 0)
self.assertEqual(len(stage.outs), 1)
self.assertEqual(stage.relpath, "foo.dvc")
self.assertEqual(len(os.listdir(dname)), 1)
self.assertTrue(os.path.isfile(fname))
self.assertTrue(filecmp.cmp(fname, "foo", shallow=False))
dvc.add("remote://workspace/file")
assert (tmp_dir / "file.dvc").read_text() == (
"outs:\n"
f"- {hash_name}: {hash_value}\n"
" path: remote://workspace/file\n"
)
assert (workspace / "file").read_text() == "file"
assert (
workspace / "cache" / hash_value[:2] / hash_value[2:]
).read_text() == "file"

assert dvc.status() == {}


@pytest.mark.parametrize(
"workspace, hash_name, hash_value",
[
(
pytest.lazy_fixture("local_cloud"),
"md5",
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
),
pytest.param(
pytest.lazy_fixture("ssh"),
"md5",
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
(
pytest.lazy_fixture("s3"),
"etag",
"ec602a6ba97b2dd07bd6d2cd89674a60.dir",
),
(
pytest.lazy_fixture("gs"),
"md5",
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
),
],
indirect=["workspace"],
)
def test_add_external_dir(tmp_dir, dvc, workspace, hash_name, hash_value):
workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})

dvc.add("remote://workspace/dir")
assert (tmp_dir / "dir.dvc").read_text() == (
"outs:\n"
f"- {hash_name}: {hash_value}\n"
" path: remote://workspace/dir\n"
)
assert (workspace / "cache" / hash_value[:2] / hash_value[2:]).is_file()


class TestAddLocalRemoteFile(TestDvc):
Expand Down
34 changes: 32 additions & 2 deletions tests/func/test_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,22 @@ def test_open(tmp_dir, dvc, remote):
assert fd.read() == "foo-text"


@pytest.mark.parametrize("cloud", clouds)
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture(cloud)
for cloud in [
"real_s3", # NOTE: moto's s3 fails in some tests
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"hdfs",
"http",
]
],
)
def test_open_external(erepo_dir, cloud):
erepo_dir.add_remote(config=cloud.config)

Expand Down Expand Up @@ -104,7 +119,22 @@ def test_open_granular(tmp_dir, dvc, remote):
assert fd.read() == "foo-text"


@pytest.mark.parametrize("remote", all_remotes)
@pytest.mark.parametrize(
"remote",
[
pytest.lazy_fixture(f"{cloud}_remote")
for cloud in [
"real_s3", # NOTE: moto's s3 fails in some tests
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"hdfs",
"http",
]
],
)
def test_missing(tmp_dir, dvc, remote):
tmp_dir.dvc_gen("foo", "foo")

Expand Down
42 changes: 42 additions & 0 deletions tests/func/test_gc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,3 +348,45 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
dvc.gc(workspace=True, force=True)
assert _count_files(dvc.cache.local.cache_dir) == 0


@pytest.mark.parametrize(
"workspace",
[
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("hdfs"),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
],
indirect=True,
)
def test_gc_external_output(tmp_dir, dvc, workspace):
workspace.gen({"foo": "foo", "bar": "bar"})

(foo_stage,) = dvc.add("remote://workspace/foo")
(bar_stage,) = dvc.add("remote://workspace/bar")

foo_hash = foo_stage.outs[0].checksum
bar_hash = bar_stage.outs[0].checksum

assert (
workspace / "cache" / foo_hash[:2] / foo_hash[2:]
).read_text() == "foo"
assert (
workspace / "cache" / bar_hash[:2] / bar_hash[2:]
).read_text() == "bar"

(tmp_dir / "foo.dvc").unlink()

dvc.gc(workspace=True)

assert not (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).exists()
assert (
workspace / "cache" / bar_hash[:2] / bar_hash[2:]
).read_text() == "bar"
53 changes: 53 additions & 0 deletions tests/func/test_import_url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,56 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
dvc.imp_url(src, ".", no_exec=True)
dst = tmp_dir / "file"
assert not dst.exists()


@pytest.mark.parametrize(
"workspace",
[
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("hdfs"),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
pytest.lazy_fixture("http"),
],
indirect=True,
)
def test_import_url(tmp_dir, dvc, workspace):
workspace.gen("file", "file")
assert not (tmp_dir / "file").exists() # sanity check
dvc.imp_url("remote://workspace/file")
assert (tmp_dir / "file").read_text() == "file"

assert dvc.status() == {}


@pytest.mark.parametrize(
"workspace",
[
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("gs"),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
],
indirect=True,
)
def test_import_url_dir(tmp_dir, dvc, workspace):
workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
assert not (tmp_dir / "dir").exists() # sanity check
dvc.imp_url("remote://workspace/dir")
assert set(os.listdir(tmp_dir / "dir")) == {"file", "subdir"}
assert (tmp_dir / "dir" / "file").read_text() == "file"
assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"]
assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile"

assert dvc.status() == {}
Loading