diff --git a/dvc/cli.py b/dvc/cli.py index 37d148d7c8..320a88ee2a 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -3,37 +3,39 @@ import logging import sys -import dvc.command.add as add -import dvc.command.cache as cache -import dvc.command.checkout as checkout -import dvc.command.commit as commit -import dvc.command.config as config -import dvc.command.daemon as daemon -import dvc.command.data_sync as data_sync -import dvc.command.destroy as destroy -import dvc.command.diff as diff -import dvc.command.gc as gc -import dvc.command.get as get -import dvc.command.get_url as get_url -import dvc.command.imp as imp -import dvc.command.imp_url as imp_url -import dvc.command.init as init -import dvc.command.install as install -import dvc.command.lock as lock -import dvc.command.metrics as metrics -import dvc.command.move as move -import dvc.command.pipeline as pipeline -import dvc.command.remote as remote -import dvc.command.remove as remove -import dvc.command.repro as repro -import dvc.command.root as root -import dvc.command.run as run -import dvc.command.tag as tag -import dvc.command.unprotect as unprotect -import dvc.command.update as update -import dvc.command.version as version -from dvc.command.base import fix_subparsers -from dvc.exceptions import DvcParserError +from .command import ( + add, + cache, + checkout, + commit, + config, + daemon, + data_sync, + destroy, + diff, + gc, + get, + get_url, + imp, + imp_url, + init, + install, + lock, + metrics, + move, + pipeline, + remote, + remove, + repro, + root, + run, + tag, + unprotect, + update, + version, +) +from .command.base import fix_subparsers +from .exceptions import DvcParserError logger = logging.getLogger(__name__) diff --git a/dvc/output/base.py b/dvc/output/base.py index ee69e30bcd..536b4950ce 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -405,7 +405,7 @@ def get_used_cache(self, **kwargs): if self.stage.is_repo_import: cache = NamedCache() - dep, = self.stage.deps + (dep,) = self.stage.deps cache.external[dep.repo_pair].add(dep.def_path) return cache diff --git a/dvc/progress.py b/dvc/progress.py index fefbd98f61..d03b037c06 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -4,7 +4,6 @@ import sys from threading import RLock -from funcy import merge from tqdm import tqdm from dvc.utils import env2bool @@ -31,7 +30,10 @@ class Tqdm(tqdm): ) BAR_FMT_NOTOTAL = ( "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}" - " [{elapsed}11}{postfix}]" + " [{elapsed}, {rate_fmt:>11}{postfix}]" + ) + BYTES_DEFAULTS = dict( + unit="B", unit_scale=True, unit_divisor=1024, miniters=1 ) def __init__( @@ -44,6 +46,7 @@ def __init__( bar_format=None, bytes=False, # pylint: disable=W0622 file=None, + total=None, **kwargs ): """ @@ -58,12 +61,10 @@ def __init__( kwargs : anything accepted by `tqdm.tqdm()` """ kwargs = kwargs.copy() - kwargs.setdefault("unit_scale", True) if bytes: - bytes_defaults = dict( - unit="B", unit_scale=True, unit_divisor=1024, miniters=1 - ) - kwargs = merge(bytes_defaults, kwargs) + kwargs = {**self.BYTES_DEFAULTS, **kwargs} + else: + kwargs.setdefault("unit_scale", total > 999 if total else True) if file is None: file = sys.stderr self.desc_persist = desc @@ -84,6 +85,7 @@ def __init__( desc=desc, bar_format="!", lock_args=(False,), + total=total, **kwargs ) if bar_format is None: @@ -114,6 +116,10 @@ def update_to(self, current, total=None): def close(self): if self.desc_persist is not None: self.set_description_str(self.desc_persist, refresh=False) + # unknown/zero ETA + self.bar_format = self.bar_format.replace("<{remaining}", "") + # remove completed bar + self.bar_format = self.bar_format.replace("|{bar:10}|", " ") super().close() @property diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 33ec566bb7..42b283ba37 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -493,7 +493,9 @@ def _save_dir(self, path_info, checksum): cache_info = self.checksum_to_path_info(checksum) dir_info = self.get_dir_cache(checksum) - for entry in dir_info: + for entry in Tqdm( + dir_info, desc="Saving " + path_info.name, unit="file" + ): entry_info = path_info / entry[self.PARAM_RELPATH] entry_checksum = entry[self.PARAM_CHECKSUM] self._save_file(entry_info, entry_checksum, save_link=False) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 9d0a506a87..bc90b89e77 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -205,7 +205,7 @@ def collect_granular(self, target, *args, **kwargs): return [(stage, None) for stage in self.stages] try: - out, = self.find_outs_by_path(target, strict=False) + (out,) = self.find_outs_by_path(target, strict=False) filter_info = PathInfo(os.path.abspath(target)) return [(out.stage, filter_info)] except OutputNotFoundError: @@ -419,7 +419,7 @@ def func(out): def find_out_by_relpath(self, relpath): path = os.path.join(self.root_dir, relpath) - out, = self.find_outs_by_path(path) + (out,) = self.find_outs_by_path(path) return out def is_dvc_internal(self, path): diff --git a/dvc/repo/add.py b/dvc/repo/add.py index 51fc417f8e..e469119b93 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -4,11 +4,12 @@ import colorama from . import locked -from dvc.exceptions import RecursiveAddingWhileUsingFilename -from dvc.progress import Tqdm -from dvc.repo.scm_context import scm_context -from dvc.stage import Stage -from dvc.utils import LARGE_DIR_SIZE +from ..exceptions import RecursiveAddingWhileUsingFilename +from ..output.base import OutputDoesNotExistError +from ..progress import Tqdm +from ..repo.scm_context import scm_context +from ..stage import Stage +from ..utils import LARGE_DIR_SIZE logger = logging.getLogger(__name__) @@ -23,7 +24,12 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None): targets = [targets] stages_list = [] - with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar: + num_targets = len(targets) + with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar: + if num_targets == 1: + # clear unneeded top-level progress bar for single target + pbar.bar_format = "Adding..." + pbar.refresh() for target in targets: sub_targets = _find_all_targets(repo, target, recursive) pbar.total += len(sub_targets) - 1 @@ -45,17 +51,29 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None): repo.check_modified_graph(stages) - for stage in stages: - stage.save() - - if not no_commit: - stage.commit() - - stage.dump() + with Tqdm( + total=len(stages), + desc="Processing", + unit="file", + disable=True if len(stages) == 1 else None, + ) as pbar_stages: + for stage in stages: + try: + stage.save() + except OutputDoesNotExistError: + pbar.n -= 1 + raise + + if not no_commit: + stage.commit() + + stage.dump() + pbar_stages.update() stages_list += stages - # remove filled bar bit of progress, leaving stats - pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ") + + if num_targets == 1: # restore bar format for stats + pbar.bar_format = pbar.BAR_FMT_DEFAULT return stages_list @@ -64,7 +82,12 @@ def _find_all_targets(repo, target, recursive): if os.path.isdir(target) and recursive: return [ fname - for fname in repo.tree.walk_files(target) + for fname in Tqdm( + repo.tree.walk_files(target), + desc="Searching " + target, + bar_format=Tqdm.BAR_FMT_NOTOTAL, + unit="file", + ) if not repo.is_dvc_internal(fname) if not Stage.is_stage_file(fname) if not repo.scm.belongs_to_scm(fname) @@ -76,7 +99,12 @@ def _find_all_targets(repo, target, recursive): def _create_stages(repo, targets, fname, pbar=None): stages = [] - for out in targets: + for out in Tqdm( + targets, + desc="Creating DVC-files", + disable=True if len(targets) < LARGE_DIR_SIZE else None, + unit="file", + ): stage = Stage.create(repo, outs=[out], add=True, fname=fname) if not stage: diff --git a/dvc/repo/get_url.py b/dvc/repo/get_url.py index fbc173b640..5718b20ecd 100644 --- a/dvc/repo/get_url.py +++ b/dvc/repo/get_url.py @@ -14,6 +14,6 @@ def get_url(url, out=None): out = os.path.abspath(out) - dep, = dependency.loads_from(None, [url]) - out, = output.loads_from(None, [out], use_cache=False) + (dep,) = dependency.loads_from(None, [url]) + (out,) = output.loads_from(None, [out], use_cache=False) dep.download(out)