From 21bafccc96b9c442d85d3d8080195ade0022dac5 Mon Sep 17 00:00:00 2001 From: kurianbenoy Date: Thu, 31 Oct 2019 19:11:35 +0530 Subject: [PATCH 1/4] Remove multi-threaded prefix before progress bar(#2686) --- dvc/progress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/progress.py b/dvc/progress.py index 7e85b6e18c..bc3b177517 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -20,7 +20,7 @@ def __enter__(self): Creates a blank initial dummy progress bar if needed so that workers are forced to create "nested" bars. """ - blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False) + blank_bar = Tqdm(bar_format="", leave=False) if blank_bar.pos > 0: # already nested - don't need a placeholder bar blank_bar.close() From 306c916626a62237c2ef7de73e61d719fd0d8e83 Mon Sep 17 00:00:00 2001 From: kurianbenoy Date: Thu, 31 Oct 2019 19:16:05 +0530 Subject: [PATCH 2/4] Change import order in progress file --- dvc/progress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/progress.py b/dvc/progress.py index bc3b177517..24013c637a 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -2,8 +2,8 @@ from __future__ import print_function import logging import sys -from tqdm import tqdm from concurrent.futures import ThreadPoolExecutor +from tqdm import tqdm from funcy import merge from dvc.utils import env2bool From 82dae6c25b86320d75c69761541e3c580a6ab913 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 1 Nov 2019 16:06:00 +0000 Subject: [PATCH 3/4] remove special TqdmThreadPoolExecutor logic - fixes #2686 - closes #2703 --- dvc/progress.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 24013c637a..3344fff29e 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -2,34 +2,13 @@ from __future__ import print_function import logging import sys -from concurrent.futures import ThreadPoolExecutor from tqdm import tqdm +from concurrent.futures import ThreadPoolExecutor from funcy import merge from dvc.utils import env2bool logger = logging.getLogger(__name__) - - -class TqdmThreadPoolExecutor(ThreadPoolExecutor): - """ - Ensure worker progressbars are cleared away properly. - """ - - def __enter__(self): - """ - Creates a blank initial dummy progress bar if needed so that workers - are forced to create "nested" bars. - """ - blank_bar = Tqdm(bar_format="", leave=False) - if blank_bar.pos > 0: - # already nested - don't need a placeholder bar - blank_bar.close() - self.bar = blank_bar - return super(TqdmThreadPoolExecutor, self).__enter__() - - def __exit__(self, *a, **k): - super(TqdmThreadPoolExecutor, self).__exit__(*a, **k) - self.bar.close() +TqdmThreadPoolExecutor = ThreadPoolExecutor class Tqdm(tqdm): From 440b323e39b82cb646a8efd5f6ef1f867152b264 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 2 Nov 2019 15:22:39 +0000 Subject: [PATCH 4/4] TqdmThreadPoolExecutor => ThreadPoolExecutor - Fixes https://github.com/iterative/dvc/pull/2703#discussion_r341763150 --- dvc/progress.py | 2 -- dvc/remote/local.py | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 3344fff29e..68c6ddd594 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -3,12 +3,10 @@ import logging import sys from tqdm import tqdm -from concurrent.futures import ThreadPoolExecutor from funcy import merge from dvc.utils import env2bool logger = logging.getLogger(__name__) -TqdmThreadPoolExecutor = ThreadPoolExecutor class Tqdm(tqdm): diff --git a/dvc/remote/local.py b/dvc/remote/local.py index dff4337566..6c1d37d039 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -4,6 +4,7 @@ import errno import logging from functools import partial +from concurrent.futures import ThreadPoolExecutor from shortuuid import uuid @@ -29,7 +30,7 @@ ) from dvc.config import Config from dvc.exceptions import DvcException, DownloadError, UploadError -from dvc.progress import Tqdm, TqdmThreadPoolExecutor +from dvc.progress import Tqdm from dvc.path_info import PathInfo @@ -359,7 +360,7 @@ def _process( return 0 if jobs > 1: - with TqdmThreadPoolExecutor(max_workers=jobs) as executor: + with ThreadPoolExecutor(max_workers=jobs) as executor: fails = sum(executor.map(func, *plans)) else: fails = sum(map(func, *plans))