diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 1d97897904..17379d3a6f 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -6,7 +6,7 @@ from funcy.py3 import lmap, retry from dvc.utils import makedirs -from dvc.utils.compat import basestring, is_py2, pathlib, fspath, fspath_py35 +from dvc.utils.compat import basestring, pathlib, fspath, fspath_py35, bytes __all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"] @@ -55,13 +55,16 @@ def _gen(self, struct, prefix=None): path = (prefix or self) / name if isinstance(contents, dict): - self._gen(contents, prefix=path) + if not contents: + makedirs(path, exist_ok=True) + else: + self._gen(contents, prefix=path) else: makedirs(path.parent, exist_ok=True) - if is_py2 and isinstance(contents, str): + if isinstance(contents, bytes): path.write_bytes(contents) else: - path.write_text(contents) + path.write_text(contents, encoding="utf-8") def dvc_gen(self, struct, text="", commit=None): paths = self.gen(struct, text) diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index b09a06a78f..ee4c5f7a1f 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -1,186 +1,121 @@ -import itertools +# encoding: utf-8 +from __future__ import unicode_literals import os import shutil - import pytest from dvc.exceptions import DvcIgnoreInCollectedDirError -from dvc.ignore import DvcIgnore -from dvc.ignore import DvcIgnoreDirs -from dvc.ignore import DvcIgnoreFilter -from dvc.ignore import DvcIgnorePatterns +from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns from dvc.scm.tree import WorkingTree -from dvc.utils.compat import cast_bytes +from dvc.utils import walk_files +from dvc.utils.compat import fspath from dvc.utils.fs import get_mtime_and_size -from tests.basic_env import TestDvc -from tests.utils import to_posixpath - -class TestDvcIgnore(TestDvc): - def setUp(self): - super(TestDvcIgnore, self).setUp() - - def _get_all_paths(self): - - paths = [] - for root, dirs, files in self.dvc.tree.walk( - self.dvc.root_dir, dvcignore=self.dvc.dvcignore - ): - for dname in dirs: - paths.append(os.path.join(root, dname)) +from tests.utils import to_posixpath - for fname in files: - paths.append(os.path.join(root, fname)) - return paths +def test_ignore(tmp_dir, dvc, monkeypatch): + tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - def test_ignore_in_child_dir(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "w") as fobj: - fobj.write("data_dir/data") + assert _files_set("dir", dvc.dvcignore) == {"dir/other"} - forbidden_path = os.path.join(self.dvc.root_dir, self.DATA) - all_paths = self._get_all_paths() + monkeypatch.chdir("dir") + assert _files_set(".", dvc.dvcignore) == {"./other"} - self.assertNotIn(forbidden_path, all_paths) - def test_ignore_in_child_dir_unicode(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "wb") as fobj: - fobj.write(cast_bytes(self.UNICODE, "utf-8")) +def test_ignore_unicode(tmp_dir, dvc): + tmp_dir.gen({"dir": {"other": "text"}}) + # Path() doesn't handle unicode paths in Windows/Python 2 + # I don't know to debug it further, waiting till Python 2 EOL + with open("dir/тест", "wb") as fd: + fd.write("проверка".encode("utf-8")) - forbidden_path = os.path.join(self.dvc.root_dir, self.UNICODE) - all_paths = self._get_all_paths() + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") - self.assertNotIn(forbidden_path, all_paths) + assert _files_set("dir", dvc.dvcignore) == {"dir/other"} - def test_ignore_in_parent_dir(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "w") as fobj: - fobj.write("data_dir/data") - os.chdir(self.DATA_DIR) +def test_rename_ignored_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) - forbidden_path = os.path.join(self.dvc.root_dir, self.DATA) - all_paths = self._get_all_paths() + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*") + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - self.assertNotIn(forbidden_path, all_paths) + shutil.move("dir/ignored", "dir/ignored_new") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) + assert new_mtime == mtime and new_size == size -def test_metadata_unchanged_when_moving_ignored_file(dvc_repo, repo_dir): - new_data_path = repo_dir.DATA_SUB + "_new" - ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create( - ignore_file, - "\n".join( - [to_posixpath(repo_dir.DATA_SUB), to_posixpath(new_data_path)] - ), - ) +def test_rename_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - mtime_sig, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) + shutil.move("dir/foo", "dir/foo_new") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - shutil.move(repo_dir.DATA_SUB, new_data_path) + assert new_mtime != mtime and new_size == size - new_mtime_sig, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) - assert new_mtime_sig == mtime_sig - assert new_size == size +def test_remove_ignored_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) -def test_mtime_changed_when_moving_non_ignored_file(dvc_repo, repo_dir): - new_data_path = repo_dir.DATA_SUB + "_new" - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) + os.remove("dir/ignored") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - shutil.move(repo_dir.DATA_SUB, new_data_path) - new_mtime, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + assert new_mtime == mtime and new_size == size - assert new_mtime != mtime - assert new_size == size +def test_remove_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) -def test_metadata_unchanged_on_ignored_file_deletion(dvc_repo, repo_dir): - ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create(ignore_file, to_posixpath(repo_dir.DATA_SUB)) + os.remove("dir/foo") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) + assert new_mtime != mtime and new_size != size - os.remove(repo_dir.DATA_SUB) - new_mtime, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) - assert new_mtime == mtime - assert new_size == size +def test_dvcignore_in_out_dir(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}}) + with pytest.raises(DvcIgnoreInCollectedDirError): + dvc.add("dir") -def test_metadata_changed_on_non_ignored_file_deletion(dvc_repo, repo_dir): - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) - os.remove(repo_dir.DATA_SUB) - new_mtime_sig, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) +# @efiop why do we need to parametrize this? +@pytest.mark.parametrize("dname", ["dir", "dir/subdir"]) +def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): + tmp_dir.gen({"dir": {"subdir": {}}}) - assert new_mtime_sig != mtime - assert new_size != size + top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE) + top_ignore_file.write_text(os.path.basename(dname)) + ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE + ignore_file.write_text("foo") -def test_should_raise_on_dvcignore_in_out_dir(dvc_repo, repo_dir): - ignore_file = os.path.join(repo_dir.DATA_DIR, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create(ignore_file, "") + assert dvc.dvcignore.ignores == { + DvcIgnoreDirs([".git", ".hg", ".dvc"]), + DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)), + } - with pytest.raises(DvcIgnoreInCollectedDirError): - dvc_repo.add(repo_dir.DATA_DIR) +def test_ignore_on_branch(tmp_dir, scm, dvc): + tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files") -@pytest.mark.parametrize("dname", [TestDvc.DATA_DIR, TestDvc.DATA_SUB_DIR]) -def test_ignore_collecting_dvcignores(repo_dir, dname): - top_ignore_file = os.path.join( - repo_dir.root_dir, os.path.dirname(dname), DvcIgnore.DVCIGNORE_FILE - ) - repo_dir.create(top_ignore_file, os.path.basename(dname)) + scm.checkout("branch", create_new=True) + tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore") - ignore_file = os.path.join( - repo_dir.root_dir, dname, DvcIgnore.DVCIGNORE_FILE - ) - repo_dir.create(ignore_file, repo_dir.FOO) + scm.checkout("master") + assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"} - assert DvcIgnoreFilter( - repo_dir.root_dir, WorkingTree(repo_dir.root_dir) - ).ignores == { - DvcIgnoreDirs([".git", ".hg", ".dvc"]), - DvcIgnorePatterns(top_ignore_file, WorkingTree(repo_dir.root_dir)), - } + dvc.tree = scm.get_tree("branch") + assert _files_set(".", dvc.dvcignore) == {"./bar"} -def test_ignore_on_branch(git, dvc_repo, repo_dir): - dvc_repo.add(repo_dir.DATA_DIR) - dvc_repo.scm.commit("add data dir") - - branch_name = "branch_one" - dvc_repo.scm.checkout(branch_name, create_new=True) - - repo_dir.create(DvcIgnore.DVCIGNORE_FILE, to_posixpath(repo_dir.DATA_SUB)) - dvc_repo.scm.add([DvcIgnore.DVCIGNORE_FILE]) - git.index.commit("add ignore") - - dvc_repo.scm.checkout("master") - - git_tree = dvc_repo.scm.get_tree(branch_name) - branch_data_files = set( - itertools.chain.from_iterable( - [ - files - for _, _, files in dvc_repo.tree.walk( - repo_dir.DATA_DIR, - dvcignore=DvcIgnoreFilter(repo_dir.root_dir, git_tree), - ) - ] - ) - ) - assert branch_data_files == {"data"} +def _files_set(root, dvcignore): + return {to_posixpath(f) for f in walk_files(root, dvcignore)}