diff --git a/dvc/commands/live.py b/dvc/commands/live.py index 221fed114f..d947ee897b 100644 --- a/dvc/commands/live.py +++ b/dvc/commands/live.py @@ -1,5 +1,4 @@ import argparse -from pathlib import Path from dvc.cli.command import CmdBase from dvc.cli.utils import fix_subparsers @@ -16,6 +15,8 @@ def _run(self, target, revs=None): metrics, plots = self.repo.live.show(target=target, revs=revs) if plots: + from pathlib import Path + html_path = Path.cwd() / (self.args.target + "_html") renderers = match_renderers(plots, self.repo.plots.templates) diff --git a/dvc/fs/_callback.py b/dvc/fs/_callback.py new file mode 100644 index 0000000000..54720bda75 --- /dev/null +++ b/dvc/fs/_callback.py @@ -0,0 +1,49 @@ +import fsspec + +from dvc.progress import Tqdm + + +class FsspecCallback(fsspec.Callback): + def __init__(self, progress_bar): + self.progress_bar = progress_bar + super().__init__() + + def set_size(self, size): + if size is not None: + self.progress_bar.total = size + self.progress_bar.refresh() + super().set_size(size) + + def relative_update(self, inc=1): + self.progress_bar.update(inc) + super().relative_update(inc) + + def absolute_update(self, value): + self.progress_bar.update_to(value) + super().absolute_update(value) + + @staticmethod + def wrap_fn(cb, fn): + def wrapped(*args, **kwargs): + res = fn(*args, **kwargs) + cb.relative_update() + return res + + return wrapped + + +def tdqm_or_callback_wrapped( + fobj, method, total, callback=None, **pbar_kwargs +): + if callback: + from funcy import nullcontext + from tqdm.utils import CallbackIOWrapper + + callback.set_size(total) + wrapper = CallbackIOWrapper(callback.relative_update, fobj, method) + return nullcontext(wrapper) + + return Tqdm.wrapattr(fobj, method, total=total, bytes=True, **pbar_kwargs) + + +DEFAULT_CALLBACK = fsspec.callbacks.NoOpCallback() diff --git a/dvc/fs/base.py b/dvc/fs/base.py index de695e6c24..edcf1207d2 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -10,7 +10,7 @@ from tqdm.utils import CallbackIOWrapper from dvc.exceptions import DvcException -from dvc.progress import DEFAULT_CALLBACK, FsspecCallback +from dvc.fs._callback import DEFAULT_CALLBACK, FsspecCallback from dvc.ui import ui from dvc.utils import tmp_fname from dvc.utils.fs import makedirs, move diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index 6322c45fd0..9d0c229af0 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -5,7 +5,7 @@ from dvc.exceptions import OutputNotFoundError from dvc.utils import relpath -from ..progress import DEFAULT_CALLBACK +from ._callback import DEFAULT_CALLBACK from ._metadata import Metadata from .base import FileSystem diff --git a/dvc/fs/fsspec_wrapper.py b/dvc/fs/fsspec_wrapper.py index 868e7da1de..d931996119 100644 --- a/dvc/fs/fsspec_wrapper.py +++ b/dvc/fs/fsspec_wrapper.py @@ -5,8 +5,7 @@ from funcy import cached_property from tqdm.utils import CallbackIOWrapper -from dvc.progress import DEFAULT_CALLBACK - +from ._callback import DEFAULT_CALLBACK from .base import FileSystem FSPath = str diff --git a/dvc/fs/http.py b/dvc/fs/http.py index dbe386d221..d19372810c 100644 --- a/dvc/fs/http.py +++ b/dvc/fs/http.py @@ -4,9 +4,9 @@ from funcy import cached_property, memoize, wrap_with from dvc import prompt -from dvc.progress import DEFAULT_CALLBACK from dvc.scheme import Schemes +from ._callback import DEFAULT_CALLBACK from .fsspec_wrapper import AnyFSPath, FSSpecWrapper, NoDirectoriesMixin diff --git a/dvc/fs/local.py b/dvc/fs/local.py index 1ddb876e07..9c7a7cb905 100644 --- a/dvc/fs/local.py +++ b/dvc/fs/local.py @@ -6,7 +6,7 @@ from dvc.utils import is_exec, tmp_fname from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove -from ..progress import DEFAULT_CALLBACK +from ._callback import DEFAULT_CALLBACK from .base import FileSystem logger = logging.getLogger(__name__) diff --git a/dvc/fs/repo.py b/dvc/fs/repo.py index 8e702377c9..9b2770676b 100644 --- a/dvc/fs/repo.py +++ b/dvc/fs/repo.py @@ -8,7 +8,7 @@ from funcy import lfilter, wrap_with -from ..progress import DEFAULT_CALLBACK +from ._callback import DEFAULT_CALLBACK from .base import FileSystem from .dvc import DvcFileSystem diff --git a/dvc/fs/s3.py b/dvc/fs/s3.py index 97e43a1c1c..7eb9632dec 100644 --- a/dvc/fs/s3.py +++ b/dvc/fs/s3.py @@ -5,9 +5,9 @@ from funcy import cached_property, wrap_prop -from dvc.progress import DEFAULT_CALLBACK from dvc.scheme import Schemes +from ._callback import DEFAULT_CALLBACK from .fsspec_wrapper import ObjectFSWrapper _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config") diff --git a/dvc/fs/ssh.py b/dvc/fs/ssh.py index 8ee5c692dc..ba069b1d13 100644 --- a/dvc/fs/ssh.py +++ b/dvc/fs/ssh.py @@ -8,7 +8,7 @@ from dvc.scheme import Schemes from dvc.utils.fs import as_atomic -from ..progress import DEFAULT_CALLBACK +from ._callback import DEFAULT_CALLBACK from .fsspec_wrapper import FSSpecWrapper _SSH_TIMEOUT = 60 * 30 diff --git a/dvc/progress.py b/dvc/progress.py index 16cab4fc90..cd5fff9a2d 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -3,7 +3,6 @@ import sys from threading import RLock -import fsspec from tqdm import tqdm from dvc.env import DVC_IGNORE_ISATTY @@ -137,6 +136,8 @@ def wrapped(*args, **kwargs): return wrapped def as_callback(self): + from dvc.fs._callback import FsspecCallback + return FsspecCallback(self) def close(self): @@ -164,49 +165,3 @@ def format_dict(self): d["ncols_desc"] = d["ncols_info"] = 1 d["prefix"] = "" return d - - -class FsspecCallback(fsspec.Callback): - def __init__(self, progress_bar): - self.progress_bar = progress_bar - super().__init__() - - def set_size(self, size): - if size is not None: - self.progress_bar.total = size - self.progress_bar.refresh() - super().set_size(size) - - def relative_update(self, inc=1): - self.progress_bar.update(inc) - super().relative_update(inc) - - def absolute_update(self, value): - self.progress_bar.update_to(value) - super().absolute_update(value) - - @staticmethod - def wrap_fn(cb, fn): - def wrapped(*args, **kwargs): - res = fn(*args, **kwargs) - cb.relative_update() - return res - - return wrapped - - -def tdqm_or_callback_wrapped( - fobj, method, total, callback=None, **pbar_kwargs -): - if callback: - from funcy import nullcontext - from tqdm.utils import CallbackIOWrapper - - callback.set_size(total) - wrapper = CallbackIOWrapper(callback.relative_update, fobj, method) - return nullcontext(wrapper) - - return Tqdm.wrapattr(fobj, method, total=total, bytes=True, **pbar_kwargs) - - -DEFAULT_CALLBACK = fsspec.callbacks.NoOpCallback() diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index fdb89e597f..f3ddcaf20f 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -200,7 +200,7 @@ def copyfile(src, dest, callback=None, no_progress_bar=False, name=None): try: System.reflink(src, dest) except OSError: - from dvc.progress import tdqm_or_callback_wrapped + from dvc.fs._callback import tdqm_or_callback_wrapped with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: with tdqm_or_callback_wrapped(