bump version, merge branch 'devel'

README.rst

@@ -380,8 +380,8 @@ Parameters
     (kilo, mega, etc.) [default: False]. If any other non-zero
     number, will scale ``total`` and ``n``.
 * dynamic_ncols  : bool, optional  
-    If set, constantly alters ``ncols`` to the environment (allowing
-    for window resizes) [default: False].
+    If set, constantly alters ``ncols`` and ``nrows`` to the
+    environment (allowing for window resizes) [default: False].
 * smoothing  : float, optional  
     Exponential moving average smoothing factor for speed estimates
     (ignored in GUI mode). Ranges from 0 (average speed) to 1
@@ -393,7 +393,7 @@ Parameters
     r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, '
     '{rate_fmt}{postfix}]'
     Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,
-    percentage, elapsed, elapsed_s, ncols, desc, unit,
+    percentage, elapsed, elapsed_s, ncols, nrows, desc, unit,
     rate, rate_fmt, rate_noinv, rate_noinv_fmt,
     rate_inv, rate_inv_fmt, postfix, unit_divisor,
     remaining, remaining_s.
@@ -419,6 +419,10 @@ Parameters
 * lock_args  : tuple, optional  
     Passed to ``refresh`` for intermediate output
     (initialisation, iterating, and updating).
+* nrows  : int, optional  
+    The screen height. If specified, hides nested bars outside this
+    bound. If unspecified, attempts to use environment height.
+    The fallback is 20.
 
 Extra CLI Options
 ~~~~~~~~~~~~~~~~~

tqdm/_version.py

@@ -5,7 +5,7 @@
 __all__ = ["__version__"]
 
 # major, minor, patch, -extra
-version_info = 4, 43, 0
+version_info = 4, 44, 0
 
 # Nice string for the version
 __version__ = '.'.join(map(str, version_info))

tqdm/contrib/concurrent.py

@@ -2,8 +2,18 @@
 Thin wrappers around `concurrent.futures`.
 """
 from __future__ import absolute_import
+from tqdm import TqdmWarning
 from tqdm.auto import tqdm as tqdm_auto
 from copy import deepcopy
+try:
+    from operator import length_hint
+except ImportError:
+    def length_hint(it, default=0):
+        """Returns `len(it)`, falling back to `default`"""
+        try:
+            return len(it)
+        except TypeError:
+            return default
 try:
     from os import cpu_count
 except ImportError:
@@ -24,19 +34,27 @@ def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs):
     Parameters
     ----------
     tqdm_class  : [default: tqdm.auto.tqdm].
+    max_workers  : [default: max(32, cpu_count() + 4)].
+    chunksize  : [default: 1].
     """
     kwargs = deepcopy(tqdm_kwargs)
     if "total" not in kwargs:
         kwargs["total"] = len(iterables[0])
     tqdm_class = kwargs.pop("tqdm_class", tqdm_auto)
     max_workers = kwargs.pop("max_workers", min(32, cpu_count() + 4))
+    chunksize = kwargs.pop("chunksize", 1)
     pool_kwargs = dict(max_workers=max_workers)
-    if sys.version_info[:2] >= (3, 7):
+    sys_version = sys.version_info[:2]
+    if sys_version >= (3, 7):
         # share lock in case workers are already using `tqdm`
         pool_kwargs.update(
             initializer=tqdm_class.set_lock, initargs=(tqdm_class.get_lock(),))
+    map_args = {}
+    if not (3, 0) < sys_version < (3, 5):
+        map_args.update(chunksize=chunksize)
     with PoolExecutor(**pool_kwargs) as ex:
-        return list(tqdm_class(ex.map(fn, *iterables), **kwargs))
+        return list(tqdm_class(
+            ex.map(fn, *iterables, **map_args), **kwargs))
 
 
 def thread_map(fn, *iterables, **tqdm_kwargs):
@@ -46,7 +64,12 @@ def thread_map(fn, *iterables, **tqdm_kwargs):
 
     Parameters
     ----------
-    tqdm_class  : [default: tqdm.auto.tqdm].
+    tqdm_class : optional
+        `tqdm` class to use for bars [default: tqdm.auto.tqdm].
+    max_workers : int, optional
+        Maximum number of workers to spawn; passed to
+        `concurrent.futures.ThreadPoolExecutor.__init__`.
+        [default: max(32, cpu_count() + 4)].
     """
     from concurrent.futures import ThreadPoolExecutor
     return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs)
@@ -59,7 +82,25 @@ def process_map(fn, *iterables, **tqdm_kwargs):
 
     Parameters
     ----------
-    tqdm_class  : [default: tqdm.auto.tqdm].
+    tqdm_class  : optional
+        `tqdm` class to use for bars [default: tqdm.auto.tqdm].
+    max_workers : int, optional
+        Maximum number of workers to spawn; passed to
+        `concurrent.futures.ProcessPoolExecutor.__init__`.
+        [default: max(32, cpu_count() + 4)].
+    chunksize : int, optional
+        Size of chunks sent to worker processes; passed to
+        `concurrent.futures.ProcessPoolExecutor.map`. [default: 1].
     """
     from concurrent.futures import ProcessPoolExecutor
+    if iterables and "chunksize" not in tqdm_kwargs:
+        # default `chunksize=1` has poor performance for large iterables
+        # (most time spent dispatching items to workers).
+        longest_iterable_len = max(map(length_hint, iterables))
+        if longest_iterable_len > 1000:
+            from warnings import warn
+            warn("Iterable length %d > 1000 but `chunksize` is not set."
+                 " This may seriously degrade multiprocess performance."
+                 " Set `chunksize=1` or more." % longest_iterable_len,
+                 TqdmWarning, stacklevel=2)
     return _executor_map(ProcessPoolExecutor, fn, *iterables, **tqdm_kwargs)

tqdm/std.py

@@ -9,7 +9,7 @@
 """
 from __future__ import absolute_import, division
 # compatibility functions and utilities
-from .utils import _supports_unicode, _environ_cols_wrapper, _range, _unich, \
+from .utils import _supports_unicode, _screen_shape_wrapper, _range, _unich, \
     _term_move_up, _unicode, WeakSet, _basestring, _OrderedDict, \
     Comparable, _is_ascii, FormatReplace, disp_len, disp_trim, \
     SimpleTextIOWrapper, CallbackIOWrapper
@@ -352,7 +352,7 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False,
             r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, '
               '{rate_fmt}{postfix}]'
             Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,
-              percentage, elapsed, elapsed_s, ncols, desc, unit,
+              percentage, elapsed, elapsed_s, ncols, nrows, desc, unit,
               rate, rate_fmt, rate_noinv, rate_noinv_fmt,
               rate_inv, rate_inv_fmt, postfix, unit_divisor,
               remaining, remaining_s.
@@ -790,6 +790,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
                  unit_scale=False, dynamic_ncols=False, smoothing=0.3,
                  bar_format=None, initial=0, position=None, postfix=None,
                  unit_divisor=1000, write_bytes=None, lock_args=None,
+                 nrows=None,
                  gui=False, **kwargs):
         """
         Parameters
@@ -852,8 +853,8 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
             (kilo, mega, etc.) [default: False]. If any other non-zero
             number, will scale `total` and `n`.
         dynamic_ncols  : bool, optional
-            If set, constantly alters `ncols` to the environment (allowing
-            for window resizes) [default: False].
+            If set, constantly alters `ncols` and `nrows` to the
+            environment (allowing for window resizes) [default: False].
         smoothing  : float, optional
             Exponential moving average smoothing factor for speed estimates
             (ignored in GUI mode). Ranges from 0 (average speed) to 1
@@ -865,7 +866,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
             r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, '
               '{rate_fmt}{postfix}]'
             Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,
-              percentage, elapsed, elapsed_s, ncols, desc, unit,
+              percentage, elapsed, elapsed_s, ncols, nrows, desc, unit,
               rate, rate_fmt, rate_noinv, rate_noinv_fmt,
               rate_inv, rate_inv_fmt, postfix, unit_divisor,
               remaining, remaining_s.
@@ -891,6 +892,10 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
         lock_args  : tuple, optional
             Passed to `refresh` for intermediate output
             (initialisation, iterating, and updating).
+        nrows  : int, optional
+            The screen height. If specified, hides nested bars outside this
+            bound. If unspecified, attempts to use environment height.
+            The fallback is 20.
         gui  : bool, optional
             WARNING: internal parameter - do not use.
             Use tqdm.gui.tqdm(...) instead. If set, will attempt to use
@@ -949,16 +954,21 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
                 TqdmKeyError("Unknown argument(s): " + str(kwargs)))
 
         # Preprocess the arguments
-        if ((ncols is None) and (file in (sys.stderr, sys.stdout))) or \
+        if ((ncols is None or nrows is None) and
+            (file in (sys.stderr, sys.stdout))) or \
                 dynamic_ncols:  # pragma: no cover
             if dynamic_ncols:
-                dynamic_ncols = _environ_cols_wrapper()
+                dynamic_ncols = _screen_shape_wrapper()
                 if dynamic_ncols:
-                    ncols = dynamic_ncols(file)
+                    ncols, nrows = dynamic_ncols(file)
             else:
-                _dynamic_ncols = _environ_cols_wrapper()
+                _dynamic_ncols = _screen_shape_wrapper()
                 if _dynamic_ncols:
-                    ncols = _dynamic_ncols(file)
+                    _ncols, _nrows = _dynamic_ncols(file)
+                    if ncols is None:
+                        ncols = _ncols
+                    if nrows is None:
+                        nrows = _nrows
 
         if miniters is None:
             miniters = 0
@@ -989,6 +999,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True,
         self.leave = leave
         self.fp = file
         self.ncols = ncols
+        self.nrows = nrows
         self.mininterval = mininterval
         self.maxinterval = maxinterval
         self.miniters = miniters
@@ -1248,8 +1259,9 @@ def close(self):
         pos = abs(self.pos)
         self._decr_instances(self)
 
-        # GUI mode
-        if not hasattr(self, "sp"):
+        # GUI mode or overflow
+        if not hasattr(self, "sp") or pos >= (self.nrows or 20):
+            # never printed so nothing to do
             return
 
         # annoyingly, _supports_unicode isn't good enough
@@ -1283,10 +1295,12 @@ def clear(self, nolock=False):
 
         if not nolock:
             self._lock.acquire()
-        self.moveto(abs(self.pos))
-        self.sp('')
-        self.fp.write('\r')  # place cursor back at the beginning of line
-        self.moveto(-abs(self.pos))
+        pos = abs(self.pos)
+        if pos < (self.nrows or 20):
+            self.moveto(pos)
+            self.sp('')
+            self.fp.write('\r')  # place cursor back at the beginning of line
+            self.moveto(-pos)
         if not nolock:
             self._lock.release()
 
@@ -1406,12 +1420,14 @@ def moveto(self, n):
     @property
     def format_dict(self):
         """Public API for read-only member access."""
+        if self.dynamic_ncols:
+            self.ncols, self.nrows = self.dynamic_ncols(self.fp)
+        ncols, nrows = self.ncols, self.nrows
         return dict(
             n=self.n, total=self.total,
             elapsed=self._time() - self.start_t
             if hasattr(self, 'start_t') else 0,
-            ncols=self.dynamic_ncols(self.fp)
-            if self.dynamic_ncols else self.ncols,
+            ncols=ncols, nrows=nrows,
             prefix=self.desc, ascii=self.ascii, unit=self.unit,
             unit_scale=self.unit_scale,
             rate=1 / self.avg_time if self.avg_time else None,
@@ -1434,6 +1450,12 @@ def display(self, msg=None, pos=None):
         if pos is None:
             pos = abs(self.pos)
 
+        nrows = self.nrows or 20
+        if pos >= nrows - 1:
+            if pos >= nrows:
+                return
+            msg = " ... (more hidden) ..."
+
         if pos:
             self.moveto(pos)
         self.sp(self.__repr__() if msg is None else msg)

tqdm/tests/tests_concurrent.py

@@ -1,6 +1,7 @@
 """
 Tests for `tqdm.contrib.concurrent`.
 """
+from warnings import catch_warnings
 from tqdm.contrib.concurrent import thread_map, process_map
 from tests_tqdm import with_setup, pretest, posttest, SkipTest, StringIO, \
     closing
@@ -34,3 +35,24 @@ def test_process_map():
             assert process_map(incr, a, file=our_file) == b
         except ImportError:
             raise SkipTest
+
+
+def test_chunksize_warning():
+    """Test contrib.concurrent.process_map chunksize warnings"""
+    try:
+        from unittest.mock import patch
+    except ImportError:
+        raise SkipTest
+
+    for iterables, should_warn in [
+        ([], False),
+        (['x'], False),
+        ([()], False),
+        (['x', ()], False),
+        (['x' * 1001], True),
+        (['x' * 100, ('x',) * 1001], True),
+    ]:
+        with patch('tqdm.contrib.concurrent._executor_map'):
+            with catch_warnings(record=True) as w:
+                process_map(incr, *iterables)
+                assert should_warn == bool(w)

tqdm/tests/tests_tqdm.py

@@ -194,8 +194,7 @@ def squash_ctrlchars(s):
     lines = ['']  # state of our fake terminal
 
     # Split input string by control codes
-    RE_ctrl = re.compile("(%s)" % ("|".join(CTRLCHR)), flags=re.DOTALL)
-    s_split = RE_ctrl.split(s)
+    s_split = RE_ctrlchr.split(s)
     s_split = filter(None, s_split)  # filter out empty splits
 
     # For each control character or message
@@ -1890,3 +1889,74 @@ def test_float_progress():
                         assert not w
                 assert w
                 assert "clamping frac" in str(w[-1].message)
+
+
+@with_setup(pretest, posttest)
+def test_screen_shape():
+    """Test screen shape"""
+    # ncols
+    with closing(StringIO()) as our_file:
+        with trange(10, file=our_file, ncols=50) as t:
+            list(t)
+
+        res = our_file.getvalue()
+        assert all(len(i.strip('\n')) in (0, 50) for i in res.split('\r'))
+
+    # no second bar, leave=False
+    with closing(StringIO()) as our_file:
+        kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0,
+                      mininterval=0, leave=False)
+        with trange(10, desc="one", **kwargs) as t1:
+            with trange(10, desc="two", **kwargs) as t2:
+                list(t2)
+            list(t1)
+
+        res = our_file.getvalue()
+        assert "one" in res
+        assert "two" not in res
+        assert "\n\n" not in res
+        assert "more hidden" in res
+        # double-check ncols
+        assert all(len(i) in (0, 50) for i in squash_ctrlchars(res)
+                   if "more hidden" not in i)
+
+    # no third bar, leave=True
+    with closing(StringIO()) as our_file:
+        kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0,
+                      mininterval=0)
+        with trange(10, desc="one", **kwargs) as t1:
+            with trange(10, desc="two", **kwargs) as t2:
+                assert "two" not in our_file.getvalue()
+                with trange(10, desc="three", **kwargs) as t3:
+                    list(t3)
+                list(t2)
+            list(t1)
+
+        res = our_file.getvalue()
+        assert "one" in res
+        assert "two" in res
+        assert "three" not in res
+        assert "\n\n" not in res
+        assert "more hidden" in res
+        # double-check ncols
+        assert all(len(i) in (0, 50) for i in squash_ctrlchars(res)
+                   if "more hidden" not in i)
+
+    # second bar becomes first, leave=False
+    with closing(StringIO()) as our_file:
+        kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0,
+                      mininterval=0, leave=False)
+        t1 = tqdm(total=10, desc="one", **kwargs)
+        t2 = tqdm(total=10, desc="two", **kwargs)
+        t1.update()
+        t2.update()
+        t1.close()
+        res = our_file.getvalue()
+        assert "one" in res
+        assert "two" not in res
+        assert "more hidden" in res
+        t2.update()
+        t2.close()
+
+        res = our_file.getvalue()
+        assert "two" in res