From c6c67ace20b2a8978b5379c61444deb0196f2ce6 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 6 Dec 2019 23:46:00 +0100 Subject: [PATCH 1/5] test: create empty dirs via tmp_dir.gen() --- tests/dir_helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 1d97897904..f58c03ed82 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -55,7 +55,10 @@ def _gen(self, struct, prefix=None): path = (prefix or self) / name if isinstance(contents, dict): - self._gen(contents, prefix=path) + if not contents: + makedirs(path, exist_ok=True) + else: + self._gen(contents, prefix=path) else: makedirs(path.parent, exist_ok=True) if is_py2 and isinstance(contents, str): From 42096c5bfc37608d04c6e4a4cacb8d677af22443 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 6 Dec 2019 23:46:20 +0100 Subject: [PATCH 2/5] test: migrate test_ignore to new helpers --- tests/func/test_ignore.py | 207 +++++++++++++------------------------- 1 file changed, 68 insertions(+), 139 deletions(-) diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index b09a06a78f..84b8af966f 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -1,186 +1,115 @@ -import itertools +# encoding: utf-8 +from __future__ import unicode_literals import os import shutil - import pytest from dvc.exceptions import DvcIgnoreInCollectedDirError -from dvc.ignore import DvcIgnore -from dvc.ignore import DvcIgnoreDirs -from dvc.ignore import DvcIgnoreFilter -from dvc.ignore import DvcIgnorePatterns +from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns from dvc.scm.tree import WorkingTree -from dvc.utils.compat import cast_bytes +from dvc.utils import walk_files +from dvc.utils.compat import fspath from dvc.utils.fs import get_mtime_and_size -from tests.basic_env import TestDvc -from tests.utils import to_posixpath - - -class TestDvcIgnore(TestDvc): - def setUp(self): - super(TestDvcIgnore, self).setUp() - - def _get_all_paths(self): - - paths = [] - for root, dirs, files in self.dvc.tree.walk( - self.dvc.root_dir, dvcignore=self.dvc.dvcignore - ): - for dname in dirs: - paths.append(os.path.join(root, dname)) - - for fname in files: - paths.append(os.path.join(root, fname)) - return paths - def test_ignore_in_child_dir(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "w") as fobj: - fobj.write("data_dir/data") +def test_ignore_in_child_dir(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - forbidden_path = os.path.join(self.dvc.root_dir, self.DATA) - all_paths = self._get_all_paths() + assert set(walk_files("dir", dvc.dvcignore)) == {"dir/other"} - self.assertNotIn(forbidden_path, all_paths) - def test_ignore_in_child_dir_unicode(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "wb") as fobj: - fobj.write(cast_bytes(self.UNICODE, "utf-8")) +def test_ignore_in_parent_dir(tmp_dir, dvc, monkeypatch): + tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - forbidden_path = os.path.join(self.dvc.root_dir, self.UNICODE) - all_paths = self._get_all_paths() + monkeypatch.chdir("dir") + assert set(walk_files(".", dvc.dvcignore)) == {"./other"} - self.assertNotIn(forbidden_path, all_paths) - def test_ignore_in_parent_dir(self): - ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE) - with open(ignore_file, "w") as fobj: - fobj.write("data_dir/data") +def test_ignore_unicode(tmp_dir, dvc): + tmp_dir.gen({"dir": {"тест": "проверка", "other": "text"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") - os.chdir(self.DATA_DIR) + assert set(walk_files("dir", dvc.dvcignore)) == {"dir/other"} - forbidden_path = os.path.join(self.dvc.root_dir, self.DATA) - all_paths = self._get_all_paths() - self.assertNotIn(forbidden_path, all_paths) +def test_rename_ignored_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*") + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) -def test_metadata_unchanged_when_moving_ignored_file(dvc_repo, repo_dir): - new_data_path = repo_dir.DATA_SUB + "_new" + shutil.move("dir/ignored", "dir/ignored_new") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create( - ignore_file, - "\n".join( - [to_posixpath(repo_dir.DATA_SUB), to_posixpath(new_data_path)] - ), - ) + assert new_mtime == mtime and new_size == size - mtime_sig, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) - shutil.move(repo_dir.DATA_SUB, new_data_path) +def test_rename_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - new_mtime_sig, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + shutil.move("dir/foo", "dir/foo_new") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - assert new_mtime_sig == mtime_sig - assert new_size == size + assert new_mtime != mtime and new_size == size -def test_mtime_changed_when_moving_non_ignored_file(dvc_repo, repo_dir): - new_data_path = repo_dir.DATA_SUB + "_new" - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) +def test_remove_ignored_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - shutil.move(repo_dir.DATA_SUB, new_data_path) - new_mtime, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - assert new_mtime != mtime - assert new_size == size + os.remove("dir/ignored") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) + assert new_mtime == mtime and new_size == size -def test_metadata_unchanged_on_ignored_file_deletion(dvc_repo, repo_dir): - ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create(ignore_file, to_posixpath(repo_dir.DATA_SUB)) - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) +def test_remove_file(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) + mtime, size = get_mtime_and_size("dir", dvc.dvcignore) - os.remove(repo_dir.DATA_SUB) - new_mtime, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + os.remove("dir/foo") + new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore) - assert new_mtime == mtime - assert new_size == size + assert new_mtime != mtime and new_size != size -def test_metadata_changed_on_non_ignored_file_deletion(dvc_repo, repo_dir): - mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore) +def test_dvcignore_in_out_dir(tmp_dir, dvc): + tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}}) - os.remove(repo_dir.DATA_SUB) - new_mtime_sig, new_size = get_mtime_and_size( - repo_dir.DATA_DIR, dvc_repo.dvcignore - ) + with pytest.raises(DvcIgnoreInCollectedDirError): + dvc.add("dir") - assert new_mtime_sig != mtime - assert new_size != size +# @efiop why do we need to parametrize this? +@pytest.mark.parametrize("dname", ["dir", "dir/subdir"]) +def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): + tmp_dir.gen({"dir": {"subdir": {}}}) -def test_should_raise_on_dvcignore_in_out_dir(dvc_repo, repo_dir): - ignore_file = os.path.join(repo_dir.DATA_DIR, DvcIgnore.DVCIGNORE_FILE) - repo_dir.create(ignore_file, "") + top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE) + top_ignore_file.write_text(os.path.basename(dname)) - with pytest.raises(DvcIgnoreInCollectedDirError): - dvc_repo.add(repo_dir.DATA_DIR) + ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE + ignore_file.write_text("foo") + assert dvc.dvcignore.ignores == { + DvcIgnoreDirs([".git", ".hg", ".dvc"]), + DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)), + } -@pytest.mark.parametrize("dname", [TestDvc.DATA_DIR, TestDvc.DATA_SUB_DIR]) -def test_ignore_collecting_dvcignores(repo_dir, dname): - top_ignore_file = os.path.join( - repo_dir.root_dir, os.path.dirname(dname), DvcIgnore.DVCIGNORE_FILE - ) - repo_dir.create(top_ignore_file, os.path.basename(dname)) - ignore_file = os.path.join( - repo_dir.root_dir, dname, DvcIgnore.DVCIGNORE_FILE - ) - repo_dir.create(ignore_file, repo_dir.FOO) +def test_ignore_on_branch(tmp_dir, scm, dvc): + tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files") - assert DvcIgnoreFilter( - repo_dir.root_dir, WorkingTree(repo_dir.root_dir) - ).ignores == { - DvcIgnoreDirs([".git", ".hg", ".dvc"]), - DvcIgnorePatterns(top_ignore_file, WorkingTree(repo_dir.root_dir)), - } + scm.checkout("branch", create_new=True) + tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore") + scm.checkout("master") + assert set(walk_files(".", dvc.dvcignore)) == {"./foo", "./bar"} -def test_ignore_on_branch(git, dvc_repo, repo_dir): - dvc_repo.add(repo_dir.DATA_DIR) - dvc_repo.scm.commit("add data dir") - - branch_name = "branch_one" - dvc_repo.scm.checkout(branch_name, create_new=True) - - repo_dir.create(DvcIgnore.DVCIGNORE_FILE, to_posixpath(repo_dir.DATA_SUB)) - dvc_repo.scm.add([DvcIgnore.DVCIGNORE_FILE]) - git.index.commit("add ignore") - - dvc_repo.scm.checkout("master") - - git_tree = dvc_repo.scm.get_tree(branch_name) - branch_data_files = set( - itertools.chain.from_iterable( - [ - files - for _, _, files in dvc_repo.tree.walk( - repo_dir.DATA_DIR, - dvcignore=DvcIgnoreFilter(repo_dir.root_dir, git_tree), - ) - ] - ) - ) - assert branch_data_files == {"data"} + dvc.tree = scm.get_tree("branch") + assert set(walk_files(".", dvc.dvcignore)) == {"./bar"} From 77fcd75b7da5c7d29e6489071e500087748d3e46 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Sat, 7 Dec 2019 13:16:42 +0100 Subject: [PATCH 3/5] test: use utf-8 in tmp_dir.gen() --- tests/dir_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index f58c03ed82..336fd3474e 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -64,7 +64,7 @@ def _gen(self, struct, prefix=None): if is_py2 and isinstance(contents, str): path.write_bytes(contents) else: - path.write_text(contents) + path.write_text(contents, encoding="utf-8") def dvc_gen(self, struct, text="", commit=None): paths = self.gen(struct, text) From 79902988fcead014e9d532c82c1310085f087707 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Sat, 7 Dec 2019 13:17:08 +0100 Subject: [PATCH 4/5] test: fix files set comparisons in dvcignore tests under windows --- tests/func/test_ignore.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 84b8af966f..b08d57415a 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -11,27 +11,24 @@ from dvc.utils.compat import fspath from dvc.utils.fs import get_mtime_and_size - -def test_ignore_in_child_dir(tmp_dir, dvc): - tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) - tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - - assert set(walk_files("dir", dvc.dvcignore)) == {"dir/other"} +from tests.utils import to_posixpath -def test_ignore_in_parent_dir(tmp_dir, dvc, monkeypatch): +def test_ignore(tmp_dir, dvc, monkeypatch): tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") + assert _files_set("dir", dvc.dvcignore) == {"dir/other"} + monkeypatch.chdir("dir") - assert set(walk_files(".", dvc.dvcignore)) == {"./other"} + assert _files_set(".", dvc.dvcignore) == {"./other"} def test_ignore_unicode(tmp_dir, dvc): tmp_dir.gen({"dir": {"тест": "проверка", "other": "text"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") - assert set(walk_files("dir", dvc.dvcignore)) == {"dir/other"} + assert _files_set("dir", dvc.dvcignore) == {"dir/other"} def test_rename_ignored_file(tmp_dir, dvc): @@ -109,7 +106,11 @@ def test_ignore_on_branch(tmp_dir, scm, dvc): tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore") scm.checkout("master") - assert set(walk_files(".", dvc.dvcignore)) == {"./foo", "./bar"} + assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"} dvc.tree = scm.get_tree("branch") - assert set(walk_files(".", dvc.dvcignore)) == {"./bar"} + assert _files_set(".", dvc.dvcignore) == {"./bar"} + + +def _files_set(root, dvcignore): + return {to_posixpath(f) for f in walk_files(root, dvcignore)} From 6c12a2e555ab75ecd81a37723047e969e0178519 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Sat, 7 Dec 2019 15:50:03 +0100 Subject: [PATCH 5/5] test: fix unicode test_ignore issue under Windows Py2 --- tests/dir_helpers.py | 4 ++-- tests/func/test_ignore.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 336fd3474e..17379d3a6f 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -6,7 +6,7 @@ from funcy.py3 import lmap, retry from dvc.utils import makedirs -from dvc.utils.compat import basestring, is_py2, pathlib, fspath, fspath_py35 +from dvc.utils.compat import basestring, pathlib, fspath, fspath_py35, bytes __all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"] @@ -61,7 +61,7 @@ def _gen(self, struct, prefix=None): self._gen(contents, prefix=path) else: makedirs(path.parent, exist_ok=True) - if is_py2 and isinstance(contents, str): + if isinstance(contents, bytes): path.write_bytes(contents) else: path.write_text(contents, encoding="utf-8") diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index b08d57415a..ee4c5f7a1f 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -25,7 +25,12 @@ def test_ignore(tmp_dir, dvc, monkeypatch): def test_ignore_unicode(tmp_dir, dvc): - tmp_dir.gen({"dir": {"тест": "проверка", "other": "text"}}) + tmp_dir.gen({"dir": {"other": "text"}}) + # Path() doesn't handle unicode paths in Windows/Python 2 + # I don't know to debug it further, waiting till Python 2 EOL + with open("dir/тест", "wb") as fd: + fd.write("проверка".encode("utf-8")) + tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") assert _files_set("dir", dvc.dvcignore) == {"dir/other"}