From 94668a838f2193f25f4840fc71d216a3f012b8f7 Mon Sep 17 00:00:00 2001 From: Anatoly Rubanov Date: Fri, 17 Jul 2020 15:04:24 +0300 Subject: [PATCH 1/9] add special folder warning --- dvc/ignore.py | 32 ++++++++++++++++++++++++++++++++ tests/func/test_ignore.py | 24 +++++++++++++++++++++--- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/dvc/ignore.py b/dvc/ignore.py index 2ab78ea00a..8753cfc84a 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -150,6 +150,37 @@ def __eq__(self, other): return self.basenames == other.basenames +class DvcWarningDirs: + special_dirs = { + ".petest_cache", + ".mypy_cache", + "node_modules", + ".venv", + "venv", + "env", + "__pycache__", + "dist", + "eggs", + "wheels", + "htmlcov", + "docs", + ".ipynb_checkpoints", + ".vscode", + ".github", + } + + def __call__(self, root, dirs, files): + for d in dirs: + if d in self.special_dirs: + print( + f"Warning: {d} found in dvc, traversing may be slow." + f" You can add it in .dvcignore. See more " + f"https://dvc.org/doc/user-guide/dvcignore" + ) + + return dirs, files + + class DvcIgnoreRepo(DvcIgnore): def __call__(self, root, dirs, files): def is_dvc_repo(directory): @@ -169,6 +200,7 @@ def __init__(self, tree, root_dir): self.ignores = { DvcIgnoreDirs([".git", ".hg", ".dvc"]), DvcIgnoreRepo(), + DvcWarningDirs(), } ignore_pattern_trie = DvcIgnorePatternsTrie() for root, dirs, _ in self.tree.walk(self.root_dir): diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 31b66e047f..46bc63c74e 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -10,6 +10,7 @@ DvcIgnorePatterns, DvcIgnorePatternsTrie, DvcIgnoreRepo, + DvcWarningDirs, ) from dvc.path_info import PathInfo from dvc.repo import Repo @@ -98,7 +99,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE ignore_file.write_text("foo") - assert len(dvc.tree.dvcignore.ignores) == 3 + assert len(dvc.tree.dvcignore.ignores) == 4 assert DvcIgnoreDirs([".git", ".hg", ".dvc"]) in dvc.tree.dvcignore.ignores ignore_pattern_trie = None for ignore in dvc.tree.dvcignore.ignores: @@ -152,7 +153,7 @@ def test_match_nested(tmp_dir, dvc): assert result == {".dvcignore", "foo"} -def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory): +def test_ignore_external(tmp_dir, dvc, tmp_path_factory): tmp_dir.gen(".dvcignore", "*.backup\ntmp") ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir"))) ext_dir.gen({"y.backup": "y", "tmp": "ext tmp"}) @@ -248,7 +249,7 @@ def test_ignore_directory(tmp_dir, dvc): } -def test_multi_ignore_file(tmp_dir, dvc, monkeypatch): +def test_multi_ignore_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore") tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}}) @@ -345,3 +346,20 @@ def test_pattern_trie_tree(tmp_dir, dvc): ) == ignore_pattern_bottom ) + + +def test_special_folder_warning(tmp_dir, dvc, capsys): + for d in DvcWarningDirs.special_dirs: + tmp_dir.gen({"dir": {"tmp": "content"}, d: {"file": "test"}}) + + for _ in dvc.tree.walk_files("."): + pass + + captured = capsys.readouterr() + output = captured.out + message = ( + f"Warning: {d} found in dvc, traversing will be slow." + f" You can add it in .dvcignore. See " + f"more https://dvc.org/doc/user-guide/dvcignore" + ) + assert message in output From 8686f0d052d41ad62db5286cb311fdfa321d2be4 Mon Sep 17 00:00:00 2001 From: Anatoly Rubanov Date: Fri, 17 Jul 2020 15:04:24 +0300 Subject: [PATCH 2/9] add special folder warning --- dvc/ignore.py | 32 ++++++++++++++++++++++++++++++++ tests/func/test_ignore.py | 24 +++++++++++++++++++++--- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/dvc/ignore.py b/dvc/ignore.py index f72623295e..34d0193874 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -148,6 +148,37 @@ def __eq__(self, other): return self.basenames == other.basenames +class DvcWarningDirs: + special_dirs = { + ".petest_cache", + ".mypy_cache", + "node_modules", + ".venv", + "venv", + "env", + "__pycache__", + "dist", + "eggs", + "wheels", + "htmlcov", + "docs", + ".ipynb_checkpoints", + ".vscode", + ".github", + } + + def __call__(self, root, dirs, files): + for d in dirs: + if d in self.special_dirs: + print( + f"Warning: {d} found in dvc, traversing may be slow." + f" You can add it in .dvcignore. See more " + f"https://dvc.org/doc/user-guide/dvcignore" + ) + + return dirs, files + + class DvcIgnoreRepo(DvcIgnore): def __call__(self, root, dirs, files): def is_dvc_repo(directory): @@ -181,6 +212,7 @@ def __init__(self, tree, root_dir): self.ignores = { DvcIgnoreDirs([".git", ".hg", ".dvc"]), DvcIgnoreRepo(), + DvcWarningDirs(), } ignore_pattern_trie = DvcIgnorePatternsTrie() for root, dirs, _ in self.tree.walk(self.root_dir): diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 41533ac389..74b6a41c79 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -10,6 +10,7 @@ DvcIgnorePatterns, DvcIgnorePatternsTrie, DvcIgnoreRepo, + DvcWarningDirs, ) from dvc.path_info import PathInfo from dvc.repo import Repo @@ -103,7 +104,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE ignore_file.write_text("foo") - assert len(dvc.tree.dvcignore.ignores) == 3 + assert len(dvc.tree.dvcignore.ignores) == 4 assert DvcIgnoreDirs([".git", ".hg", ".dvc"]) in dvc.tree.dvcignore.ignores ignore_pattern_trie = None for ignore in dvc.tree.dvcignore.ignores: @@ -158,7 +159,7 @@ def test_match_nested(tmp_dir, dvc): assert result == {".dvcignore", "foo"} -def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory): +def test_ignore_external(tmp_dir, dvc, tmp_path_factory): tmp_dir.gen(".dvcignore", "*.backup\ntmp") ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir"))) ext_dir.gen({"y.backup": "y", "tmp": "ext tmp"}) @@ -265,7 +266,7 @@ def test_ignore_directory(tmp_dir, dvc): } -def test_multi_ignore_file(tmp_dir, dvc, monkeypatch): +def test_multi_ignore_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore") tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}}) @@ -389,3 +390,20 @@ def test_ignore_in_added_dir(tmp_dir, dvc): dvc.checkout() assert not ignored_path.exists() + + +def test_special_folder_warning(tmp_dir, dvc, capsys): + for d in DvcWarningDirs.special_dirs: + tmp_dir.gen({"dir": {"tmp": "content"}, d: {"file": "test"}}) + + for _ in dvc.tree.walk_files("."): + pass + + captured = capsys.readouterr() + output = captured.out + message = ( + f"Warning: {d} found in dvc, traversing may be slow." + f" You can add it in .dvcignore. See more " + f"https://dvc.org/doc/user-guide/dvcignore" + ) + assert message in output From 7dde434a322755f72e96258c2b61b2a5e9506022 Mon Sep 17 00:00:00 2001 From: Anatoly Rubanov Date: Sat, 25 Jul 2020 18:36:26 +0500 Subject: [PATCH 3/9] remove special folder warning, start working on default dvcignore --- dvc/ignore.py | 32 -------------------------------- tests/func/test_ignore.py | 24 +++--------------------- 2 files changed, 3 insertions(+), 53 deletions(-) diff --git a/dvc/ignore.py b/dvc/ignore.py index 34d0193874..f72623295e 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -148,37 +148,6 @@ def __eq__(self, other): return self.basenames == other.basenames -class DvcWarningDirs: - special_dirs = { - ".petest_cache", - ".mypy_cache", - "node_modules", - ".venv", - "venv", - "env", - "__pycache__", - "dist", - "eggs", - "wheels", - "htmlcov", - "docs", - ".ipynb_checkpoints", - ".vscode", - ".github", - } - - def __call__(self, root, dirs, files): - for d in dirs: - if d in self.special_dirs: - print( - f"Warning: {d} found in dvc, traversing may be slow." - f" You can add it in .dvcignore. See more " - f"https://dvc.org/doc/user-guide/dvcignore" - ) - - return dirs, files - - class DvcIgnoreRepo(DvcIgnore): def __call__(self, root, dirs, files): def is_dvc_repo(directory): @@ -212,7 +181,6 @@ def __init__(self, tree, root_dir): self.ignores = { DvcIgnoreDirs([".git", ".hg", ".dvc"]), DvcIgnoreRepo(), - DvcWarningDirs(), } ignore_pattern_trie = DvcIgnorePatternsTrie() for root, dirs, _ in self.tree.walk(self.root_dir): diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 74b6a41c79..41533ac389 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -10,7 +10,6 @@ DvcIgnorePatterns, DvcIgnorePatternsTrie, DvcIgnoreRepo, - DvcWarningDirs, ) from dvc.path_info import PathInfo from dvc.repo import Repo @@ -104,7 +103,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE ignore_file.write_text("foo") - assert len(dvc.tree.dvcignore.ignores) == 4 + assert len(dvc.tree.dvcignore.ignores) == 3 assert DvcIgnoreDirs([".git", ".hg", ".dvc"]) in dvc.tree.dvcignore.ignores ignore_pattern_trie = None for ignore in dvc.tree.dvcignore.ignores: @@ -159,7 +158,7 @@ def test_match_nested(tmp_dir, dvc): assert result == {".dvcignore", "foo"} -def test_ignore_external(tmp_dir, dvc, tmp_path_factory): +def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory): tmp_dir.gen(".dvcignore", "*.backup\ntmp") ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir"))) ext_dir.gen({"y.backup": "y", "tmp": "ext tmp"}) @@ -266,7 +265,7 @@ def test_ignore_directory(tmp_dir, dvc): } -def test_multi_ignore_file(tmp_dir, dvc): +def test_multi_ignore_file(tmp_dir, dvc, monkeypatch): tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore") tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}}) @@ -390,20 +389,3 @@ def test_ignore_in_added_dir(tmp_dir, dvc): dvc.checkout() assert not ignored_path.exists() - - -def test_special_folder_warning(tmp_dir, dvc, capsys): - for d in DvcWarningDirs.special_dirs: - tmp_dir.gen({"dir": {"tmp": "content"}, d: {"file": "test"}}) - - for _ in dvc.tree.walk_files("."): - pass - - captured = capsys.readouterr() - output = captured.out - message = ( - f"Warning: {d} found in dvc, traversing may be slow." - f" You can add it in .dvcignore. See more " - f"https://dvc.org/doc/user-guide/dvcignore" - ) - assert message in output From a3ee32819cc5f716ec17ca610a3ce7ad513e99e9 Mon Sep 17 00:00:00 2001 From: Anatoly Rubanov Date: Sun, 26 Jul 2020 19:55:47 +0500 Subject: [PATCH 4/9] add default .dvcignote and templates --- dvc/command/init.py | 8 ++ dvc/ignore_templates/__init__.py | 2 + dvc/ignore_templates/java | 23 ++++++ dvc/ignore_templates/julia | 24 ++++++ dvc/ignore_templates/python | 138 +++++++++++++++++++++++++++++++ dvc/ignore_templates/r | 45 ++++++++++ dvc/ignore_templates/scala | 2 + dvc/repo/__init__.py | 16 +++- dvc/repo/init.py | 27 +++++- 9 files changed, 282 insertions(+), 3 deletions(-) create mode 100644 dvc/ignore_templates/__init__.py create mode 100644 dvc/ignore_templates/java create mode 100644 dvc/ignore_templates/julia create mode 100644 dvc/ignore_templates/python create mode 100644 dvc/ignore_templates/r create mode 100644 dvc/ignore_templates/scala diff --git a/dvc/command/init.py b/dvc/command/init.py index bfd5bb88e8..79c146917e 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -17,6 +17,7 @@ def run(self): no_scm=self.args.no_scm, force=self.args.force, subdir=self.args.subdir, + ignore_template_list=self.args.ignore_template, ) self.config = self.repo.config except InitError: @@ -66,4 +67,11 @@ def add_parser(subparsers, parent_parser): "parent SCM repository." ), ) + init_parser.add_argument( + "--ignore_template", + choices=["java", "julia", "python", "r", "scala"], + nargs="+", + default=None, + help="Fill .dvcignore from template", + ) init_parser.set_defaults(func=CmdInit) diff --git a/dvc/ignore_templates/__init__.py b/dvc/ignore_templates/__init__.py new file mode 100644 index 0000000000..8747b54719 --- /dev/null +++ b/dvc/ignore_templates/__init__.py @@ -0,0 +1,2 @@ +# templates for .dvcignore files +# based on https://github.com/github/gitignore diff --git a/dvc/ignore_templates/java b/dvc/ignore_templates/java new file mode 100644 index 0000000000..a1c2a238a9 --- /dev/null +++ b/dvc/ignore_templates/java @@ -0,0 +1,23 @@ +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* diff --git a/dvc/ignore_templates/julia b/dvc/ignore_templates/julia new file mode 100644 index 0000000000..29126e47b0 --- /dev/null +++ b/dvc/ignore_templates/julia @@ -0,0 +1,24 @@ +# Files generated by invoking Julia with --code-coverage +*.jl.cov +*.jl.*.cov + +# Files generated by invoking Julia with --track-allocation +*.jl.mem + +# System-specific files and directories generated by the BinaryProvider and BinDeps packages +# They contain absolute paths specific to the host computer, and so should not be committed +deps/deps.jl +deps/build.log +deps/downloads/ +deps/usr/ +deps/src/ + +# Build artifacts for creating documentation generated by the Documenter package +docs/build/ +docs/site/ + +# File generated by Pkg, the package manager, based on a corresponding Project.toml +# It records a fixed state of all packages used by the project. As such, it should not be +# committed for packages, but should be committed for applications that require a static +# environment. +Manifest.toml diff --git a/dvc/ignore_templates/python b/dvc/ignore_templates/python new file mode 100644 index 0000000000..a81c8ee121 --- /dev/null +++ b/dvc/ignore_templates/python @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/dvc/ignore_templates/r b/dvc/ignore_templates/r new file mode 100644 index 0000000000..da10c745ee --- /dev/null +++ b/dvc/ignore_templates/r @@ -0,0 +1,45 @@ +# History files +.Rhistory +.Rapp.history + +# Session Data files +.RData + +# User-specific files +.Ruserdata + +# Example code in package build process +*-Ex.R + +# Output files from R CMD build +/*.tar.gz + +# Output files from R CMD check +/*.Rcheck/ + +# RStudio files +.Rproj.user/ + +# produced vignettes +vignettes/*.html +vignettes/*.pdf + +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 +.httr-oauth + +# knitr and R markdown default cache directories +*_cache/ +/cache/ + +# Temporary files created by R markdown +*.utf8.md +*.knit.md + +# R Environment Variables +.Renviron + +# pkgdown site +docs/ + +# translation temp files +po/*~ diff --git a/dvc/ignore_templates/scala b/dvc/ignore_templates/scala new file mode 100644 index 0000000000..9c07d4ae98 --- /dev/null +++ b/dvc/ignore_templates/scala @@ -0,0 +1,2 @@ +*.class +*.log diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 6d98682727..c57a43cf1c 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -185,10 +185,22 @@ def find_dvc_dir(cls, root=None): return os.path.join(root_dir, cls.DVC_DIR) @staticmethod - def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): + def init( + root_dir=os.curdir, + no_scm=False, + force=False, + subdir=False, + ignore_template_list=None, + ): from dvc.repo.init import init - init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir) + init( + root_dir=root_dir, + no_scm=no_scm, + force=force, + subdir=subdir, + ignore_template_list=ignore_template_list, + ) return Repo(root_dir) def unprotect(self, target): diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 695d83bcb3..0526350dc2 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -1,5 +1,6 @@ import logging import os +from importlib import resources import colorama @@ -43,7 +44,29 @@ def _welcome_message(): logger.info(msg) -def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): +def generate_dvcignore(templates_list): + with open(".dvcignore", "w") as f: + f.write( + "# This is .dvcignore file." + " See more https://dvc.org/doc/user-guide/dvcignore\n" + ) + if templates_list is not None: + for template in templates_list: + with resources.open_text( + "dvc.ignore_templates", template + ) as template_file: + f.write(f"# {template}\n") + lines = template_file.readlines() + f.writelines(lines) + + +def init( + root_dir=os.curdir, + no_scm=False, + force=False, + subdir=False, + ignore_template_list=None, +): """ Creates an empty repo on the given directory -- basically a `.dvc` directory with subdirectories for configuration and cache. @@ -103,6 +126,8 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): scm.add([config.files["repo"], proj.plot_templates.templates_dir]) + generate_dvcignore(templates_list=ignore_template_list) + if scm.ignore_file: scm.add([os.path.join(dvc_dir, scm.ignore_file)]) logger.info("\nYou can now commit the changes to git.\n") From f98d742b908ba1f4c30663a7509c4afd971422c9 Mon Sep 17 00:00:00 2001 From: Anatoly Rubanov Date: Wed, 5 Aug 2020 17:43:42 +0500 Subject: [PATCH 5/9] remove dvcignore templates --- dvc/command/init.py | 8 -- dvc/ignore_templates/__init__.py | 2 - dvc/ignore_templates/java | 23 ------ dvc/ignore_templates/julia | 24 ------ dvc/ignore_templates/python | 138 ------------------------------- dvc/ignore_templates/r | 45 ---------- dvc/ignore_templates/scala | 2 - dvc/repo/__init__.py | 14 +--- dvc/repo/init.py | 25 ++---- 9 files changed, 7 insertions(+), 274 deletions(-) delete mode 100644 dvc/ignore_templates/__init__.py delete mode 100644 dvc/ignore_templates/java delete mode 100644 dvc/ignore_templates/julia delete mode 100644 dvc/ignore_templates/python delete mode 100644 dvc/ignore_templates/r delete mode 100644 dvc/ignore_templates/scala diff --git a/dvc/command/init.py b/dvc/command/init.py index 79c146917e..bfd5bb88e8 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -17,7 +17,6 @@ def run(self): no_scm=self.args.no_scm, force=self.args.force, subdir=self.args.subdir, - ignore_template_list=self.args.ignore_template, ) self.config = self.repo.config except InitError: @@ -67,11 +66,4 @@ def add_parser(subparsers, parent_parser): "parent SCM repository." ), ) - init_parser.add_argument( - "--ignore_template", - choices=["java", "julia", "python", "r", "scala"], - nargs="+", - default=None, - help="Fill .dvcignore from template", - ) init_parser.set_defaults(func=CmdInit) diff --git a/dvc/ignore_templates/__init__.py b/dvc/ignore_templates/__init__.py deleted file mode 100644 index 8747b54719..0000000000 --- a/dvc/ignore_templates/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# templates for .dvcignore files -# based on https://github.com/github/gitignore diff --git a/dvc/ignore_templates/java b/dvc/ignore_templates/java deleted file mode 100644 index a1c2a238a9..0000000000 --- a/dvc/ignore_templates/java +++ /dev/null @@ -1,23 +0,0 @@ -# Compiled class file -*.class - -# Log file -*.log - -# BlueJ files -*.ctxt - -# Mobile Tools for Java (J2ME) -.mtj.tmp/ - -# Package Files # -*.jar -*.war -*.nar -*.ear -*.zip -*.tar.gz -*.rar - -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* diff --git a/dvc/ignore_templates/julia b/dvc/ignore_templates/julia deleted file mode 100644 index 29126e47b0..0000000000 --- a/dvc/ignore_templates/julia +++ /dev/null @@ -1,24 +0,0 @@ -# Files generated by invoking Julia with --code-coverage -*.jl.cov -*.jl.*.cov - -# Files generated by invoking Julia with --track-allocation -*.jl.mem - -# System-specific files and directories generated by the BinaryProvider and BinDeps packages -# They contain absolute paths specific to the host computer, and so should not be committed -deps/deps.jl -deps/build.log -deps/downloads/ -deps/usr/ -deps/src/ - -# Build artifacts for creating documentation generated by the Documenter package -docs/build/ -docs/site/ - -# File generated by Pkg, the package manager, based on a corresponding Project.toml -# It records a fixed state of all packages used by the project. As such, it should not be -# committed for packages, but should be committed for applications that require a static -# environment. -Manifest.toml diff --git a/dvc/ignore_templates/python b/dvc/ignore_templates/python deleted file mode 100644 index a81c8ee121..0000000000 --- a/dvc/ignore_templates/python +++ /dev/null @@ -1,138 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ diff --git a/dvc/ignore_templates/r b/dvc/ignore_templates/r deleted file mode 100644 index da10c745ee..0000000000 --- a/dvc/ignore_templates/r +++ /dev/null @@ -1,45 +0,0 @@ -# History files -.Rhistory -.Rapp.history - -# Session Data files -.RData - -# User-specific files -.Ruserdata - -# Example code in package build process -*-Ex.R - -# Output files from R CMD build -/*.tar.gz - -# Output files from R CMD check -/*.Rcheck/ - -# RStudio files -.Rproj.user/ - -# produced vignettes -vignettes/*.html -vignettes/*.pdf - -# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 -.httr-oauth - -# knitr and R markdown default cache directories -*_cache/ -/cache/ - -# Temporary files created by R markdown -*.utf8.md -*.knit.md - -# R Environment Variables -.Renviron - -# pkgdown site -docs/ - -# translation temp files -po/*~ diff --git a/dvc/ignore_templates/scala b/dvc/ignore_templates/scala deleted file mode 100644 index 9c07d4ae98..0000000000 --- a/dvc/ignore_templates/scala +++ /dev/null @@ -1,2 +0,0 @@ -*.class -*.log diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index c57a43cf1c..0e54f215d9 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -185,21 +185,11 @@ def find_dvc_dir(cls, root=None): return os.path.join(root_dir, cls.DVC_DIR) @staticmethod - def init( - root_dir=os.curdir, - no_scm=False, - force=False, - subdir=False, - ignore_template_list=None, - ): + def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): from dvc.repo.init import init init( - root_dir=root_dir, - no_scm=no_scm, - force=force, - subdir=subdir, - ignore_template_list=ignore_template_list, + root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir, ) return Repo(root_dir) diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 0526350dc2..5879cefbc4 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -1,6 +1,5 @@ import logging import os -from importlib import resources import colorama @@ -44,29 +43,15 @@ def _welcome_message(): logger.info(msg) -def generate_dvcignore(templates_list): +def generate_dvcignore(): with open(".dvcignore", "w") as f: f.write( "# This is .dvcignore file." " See more https://dvc.org/doc/user-guide/dvcignore\n" ) - if templates_list is not None: - for template in templates_list: - with resources.open_text( - "dvc.ignore_templates", template - ) as template_file: - f.write(f"# {template}\n") - lines = template_file.readlines() - f.writelines(lines) - - -def init( - root_dir=os.curdir, - no_scm=False, - force=False, - subdir=False, - ignore_template_list=None, -): + + +def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): """ Creates an empty repo on the given directory -- basically a `.dvc` directory with subdirectories for configuration and cache. @@ -126,7 +111,7 @@ def init( scm.add([config.files["repo"], proj.plot_templates.templates_dir]) - generate_dvcignore(templates_list=ignore_template_list) + generate_dvcignore() if scm.ignore_file: scm.add([os.path.join(dvc_dir, scm.ignore_file)]) From e8994dd3c658613da41c24b3647618f2b3463ae9 Mon Sep 17 00:00:00 2001 From: Anatoly Rubanov Date: Thu, 6 Aug 2020 14:17:34 +0500 Subject: [PATCH 6/9] change text and add tests --- dvc/repo/init.py | 5 +++-- tests/func/test_init.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 5879cefbc4..7817d5ca89 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -46,8 +46,9 @@ def _welcome_message(): def generate_dvcignore(): with open(".dvcignore", "w") as f: f.write( - "# This is .dvcignore file." - " See more https://dvc.org/doc/user-guide/dvcignore\n" + "# add patterns of files dvc should ignore," + " can improve performance\n" + "# Learn more at https://dvc.org/doc/user-guide/dvcignore\n" ) diff --git a/tests/func/test_init.py b/tests/func/test_init.py index 00fd8c90df..97ba998d32 100644 --- a/tests/func/test_init.py +++ b/tests/func/test_init.py @@ -101,3 +101,14 @@ def test_subdir_init_no_option(tmp_dir, scm, monkeypatch, caplog): "`--subdir` if initializing inside a subdirectory of a parent SCM " "repository.".format(os.fspath(tmp_dir / "subdir")) ) in caplog.text + + +def test_gen_dvcignore(tmp_dir): + DvcRepo.init(no_scm=True) + text = ( + "# add patterns of files dvc should ignore," + " can improve performance\n" + "# Learn more at https://dvc.org/doc/user-guide/dvcignore\n" + ) + assert (tmp_dir / ".dvcignore").exists() + assert text == (tmp_dir / ".dvcignore").read_text() From abfa2dcd1da296bc1a075fe9032dc3664045f70f Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 12 Aug 2020 23:21:28 +0300 Subject: [PATCH 7/9] refactor --- dvc/ignore.py | 15 +++++++++++++++ dvc/repo/__init__.py | 4 +--- dvc/repo/init.py | 18 ++++++------------ tests/func/test_init.py | 7 +++---- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/dvc/ignore.py b/dvc/ignore.py index 0e80a02103..2aabbb7b39 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -314,3 +314,18 @@ def check_ignore(self, target): if matches: return CheckIgnoreResult(target, True, matches) return CheckIgnoreResult(target, False, ["::"]) + + +def init(path): + dvcignore = os.path.join(path, DvcIgnore.DVCIGNORE_FILE) + if os.path.exists(dvcignore): + return dvcignore + + with open(dvcignore, "w") as fobj: + fobj.write( + "# Add patterns of files dvc should ignore, which could improve\n" + "# the performance. Learn more at\n" + "# https://dvc.org/doc/user-guide/dvcignore\n" + ) + + return dvcignore diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 0e54f215d9..6d98682727 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -188,9 +188,7 @@ def find_dvc_dir(cls, root=None): def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): from dvc.repo.init import init - init( - root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir, - ) + init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir) return Repo(root_dir) def unprotect(self, target): diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 7817d5ca89..67701c99d3 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -6,6 +6,7 @@ from dvc import analytics from dvc.config import Config from dvc.exceptions import InitError, InvalidArgumentError +from dvc.ignore import init as init_dvcignore from dvc.repo import Repo from dvc.scm import SCM from dvc.scm.base import SCMError @@ -43,15 +44,6 @@ def _welcome_message(): logger.info(msg) -def generate_dvcignore(): - with open(".dvcignore", "w") as f: - f.write( - "# add patterns of files dvc should ignore," - " can improve performance\n" - "# Learn more at https://dvc.org/doc/user-guide/dvcignore\n" - ) - - def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): """ Creates an empty repo on the given directory -- basically a @@ -108,11 +100,13 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): with config.edit() as conf: conf["core"]["no_scm"] = True - proj = Repo(root_dir) + dvcignore = init_dvcignore(root_dir) - scm.add([config.files["repo"], proj.plot_templates.templates_dir]) + proj = Repo(root_dir) - generate_dvcignore() + scm.add( + [config.files["repo"], dvcignore, proj.plot_templates.templates_dir] + ) if scm.ignore_file: scm.add([os.path.join(dvc_dir, scm.ignore_file)]) diff --git a/tests/func/test_init.py b/tests/func/test_init.py index 97ba998d32..4ea9c9101e 100644 --- a/tests/func/test_init.py +++ b/tests/func/test_init.py @@ -106,9 +106,8 @@ def test_subdir_init_no_option(tmp_dir, scm, monkeypatch, caplog): def test_gen_dvcignore(tmp_dir): DvcRepo.init(no_scm=True) text = ( - "# add patterns of files dvc should ignore," - " can improve performance\n" - "# Learn more at https://dvc.org/doc/user-guide/dvcignore\n" + "# Add patterns of files dvc should ignore, which could improve\n" + "# the performance. Learn more at\n" + "# https://dvc.org/doc/user-guide/dvcignore\n" ) - assert (tmp_dir / ".dvcignore").exists() assert text == (tmp_dir / ".dvcignore").read_text() From 50c0c846c62f17b9f43bdce83fcab82890ad6151 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 12 Aug 2020 23:54:27 +0300 Subject: [PATCH 8/9] dvcignore: ignore comments --- dvc/ignore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/ignore.py b/dvc/ignore.py index 2aabbb7b39..f6d5afae15 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -64,7 +64,7 @@ def from_files(cls, ignore_file_path, tree): for line_no, line in enumerate( map(str.strip, fobj.readlines()) ) - if line + if line and not (line.strip().startswith("#")) ] return cls(path_spec_lines, dirname) From 824fbebbbbdb5c772b9f249e349cb49f44908074 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 13 Aug 2020 00:05:56 +0300 Subject: [PATCH 9/9] adjust tests --- tests/func/test_ignore.py | 1 + tests/func/test_ls.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 67e8f71a5a..9543f4e4c5 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -132,6 +132,7 @@ def test_ignore_on_branch(tmp_dir, scm, dvc): assert set(dvc.tree.walk_files(path)) == { path / "foo", path / "bar", + path / ".dvcignore", } dvc.tree = scm.get_tree("branch", use_dvcignore=True) diff --git a/tests/func/test_ls.py b/tests/func/test_ls.py index 85dd9eb1ba..4c2681d524 100644 --- a/tests/func/test_ls.py +++ b/tests/func/test_ls.py @@ -61,6 +61,7 @@ def test_ls_repo(tmp_dir, dvc, scm): match_files( files, ( + ((".dvcignore",), False), ((".gitignore",), False), (("README.md",), False), (("structure.xml.dvc",), False), @@ -89,6 +90,7 @@ def test_ls_repo_with_color(tmp_dir, dvc, scm, mocker, monkeypatch, caplog): assert caplog.records[-1].msg == "\n".join( [ + ".dvcignore", ".gitignore", "README.md", "\x1b[01;34mdata\x1b[0m", @@ -107,6 +109,7 @@ def test_ls_repo_recursive(tmp_dir, dvc, scm): match_files( files, ( + ((".dvcignore",), False), ((".gitignore",), False), (("README.md",), False), (("structure.xml.dvc",), False), @@ -258,6 +261,7 @@ def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm): (("out.dvc",), False), (("dep",), True), (("out",), False), + ((".dvcignore",), False), ((".gitignore",), False), ), ) @@ -275,6 +279,7 @@ def test_ls_repo_with_removed_dvc_dir_recursive(tmp_dir, dvc, scm): (("out.dvc",), False), (("dep",), True), (("out", "file"), True), + ((".dvcignore",), False), ((".gitignore",), False), ), ) @@ -306,6 +311,7 @@ def test_ls_remote_repo(erepo_dir): match_files( files, ( + ((".dvcignore",), False), ((".gitignore",), False), (("README.md",), False), (("structure.xml.dvc",), False), @@ -326,6 +332,7 @@ def test_ls_remote_repo_recursive(erepo_dir): match_files( files, ( + ((".dvcignore",), False), ((".gitignore",), False), (("README.md",), False), (("structure.xml.dvc",), False), @@ -393,6 +400,7 @@ def test_ls_remote_repo_with_rev(erepo_dir): match_files( files, ( + ((".dvcignore",), False), ((".gitignore",), False), (("README.md",), False), (("model",), False), @@ -422,6 +430,7 @@ def test_ls_remote_repo_with_rev_recursive(erepo_dir): (("model", "people.csv"), True), (("model", ".gitignore"), False), (("structure.xml",), True), + ((".dvcignore",), False), ((".gitignore",), False), ), )