diff --git a/.meta/.readme.rst b/.meta/.readme.rst index 1746daa84..75d83a6aa 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -272,6 +272,16 @@ Or done on a file level using 7-zip: | grep -v Compressing 100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s] +Pre-existing CLI programs already outputting basic progress information will +benefit from ``tqdm``'s ``--update`` and ``--update_to`` flags: + +.. code:: sh + + seq 3 0.1 5 | tqdm --total 5 --update_to --null + 100%|████████████████████████████████████| 5.0/5 [00:00<00:00, 9673.21it/s] + seq 10 | tqdm --update --null # 1 + 2 + ... + 10 = 55 iterations + 55it [00:00, 90006.52it/s] + FAQ and Known Issues -------------------- diff --git a/Makefile b/Makefile index f0c5e5f8f..d9091cad0 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,7 @@ tqdm/tqdm.1: .meta/.tqdm.1.md tqdm/cli.py tqdm/std.py python -m tqdm --help | tail -n+5 |\ sed -r -e 's/\\/\\\\/g' \ -e 's/^ (--.*)=<(.*)> : (.*)$$/\n\\\1=*\2*\n: \3./' \ + -e 's/^ (--.*) : (.*)$$/\n\\\1\n: \2./' \ -e 's/ (-.*, )(--.*) /\n\1\\\2\n: /' |\ cat "$<" - |\ pandoc -o "$@" -s -t man diff --git a/README.rst b/README.rst index 5fa363e80..ac43c547b 100644 --- a/README.rst +++ b/README.rst @@ -272,6 +272,16 @@ Or done on a file level using 7-zip: | grep -v Compressing 100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s] +Pre-existing CLI programs already outputting basic progress information will +benefit from ``tqdm``'s ``--update`` and ``--update_to`` flags: + +.. code:: sh + + seq 3 0.1 5 | tqdm --total 5 --update_to --null + 100%|████████████████████████████████████| 5.0/5 [00:00<00:00, 9673.21it/s] + seq 10 | tqdm --update --null # 1 + 2 + ... + 10 = 55 iterations + 55it [00:00, 90006.52it/s] + FAQ and Known Issues -------------------- @@ -465,6 +475,14 @@ Extra CLI Options ``unit_scale`` to True, ``unit_divisor`` to 1024, and ``unit`` to 'B'. * tee : bool, optional If true, passes ``stdin`` to both ``stderr`` and ``stdout``. +* update : bool, optional + If true, will treat input as newly elapsed iterations, + i.e. numbers to pass to ``update()``. +* update_to : bool, optional + If true, will treat input as total elapsed iterations, + i.e. numbers to assign to ``self.n``. +* null : bool, optional + If true, will discard input (no stdout). * manpath : str, optional Directory in which to install tqdm man pages. * comppath : str, optional diff --git a/tqdm/cli.py b/tqdm/cli.py index 4b8f96c94..db90e7a00 100644 --- a/tqdm/cli.py +++ b/tqdm/cli.py @@ -3,14 +3,15 @@ """ from .std import tqdm, TqdmTypeError, TqdmKeyError from ._version import __version__ # NOQA -import sys -import re +from ast import literal_eval as numeric import logging +import re +import sys __all__ = ["main"] +log = logging.getLogger(__name__) def cast(val, typ): - log = logging.getLogger(__name__) log.debug((val, typ)) if " or " in typ: for t in typ.split(" or "): @@ -47,14 +48,16 @@ def isBytes(val): def posix_pipe(fin, fout, delim=b'\\n', buf_size=256, - callback=lambda int: None # pragma: no cover - ): + callback=lambda float: None, # pragma: no cover + callback_len=True): """ Params ------ fin : file with `read(buf_size : int)` method fout : file with `write` (and optionally `flush`) methods. - callback : function(int), e.g.: `tqdm.update` + callback : function(float), e.g.: `tqdm.update` + callback_len : If (default: True) do `callback(len(buffer))`. + Otherwise, do `callback(data) for data in buffer.split(delim)`. """ fp_write = fout.write @@ -92,7 +95,12 @@ def posix_pipe(fin, fout, delim=b'\\n', buf_size=256, if not tmp: if buf: fp_write(buf) - callback(1 + buf.count(delim)) # n += 1 + buf.count(delim) + if callback_len: + # n += 1 + buf.count(delim) + callback(1 + buf.count(delim)) + else: + for i in buf.split(delim): + callback(i) getattr(fout, 'flush', lambda: None)() # pragma: no cover return # n @@ -104,7 +112,8 @@ def posix_pipe(fin, fout, delim=b'\\n', buf_size=256, break else: fp_write(buf + tmp[:i + len(delim)]) - callback(1) # n += 1 + # n += 1 + callback(1 if callback_len else (buf + tmp[:i])) buf = b'' if write_bytes else '' tmp = tmp[i + len(delim):] @@ -134,6 +143,14 @@ def posix_pipe(fin, fout, delim=b'\\n', buf_size=256, `unit_scale` to True, `unit_divisor` to 1024, and `unit` to 'B'. tee : bool, optional If true, passes `stdin` to both `stderr` and `stdout`. + update : bool, optional + If true, will treat input as newly elapsed iterations, + i.e. numbers to pass to `update()`. + update_to : bool, optional + If true, will treat input as total elapsed iterations, + i.e. numbers to assign to `self.n`. + null : bool, optional + If true, will discard input (no stdout). manpath : str, optional Directory in which to install tqdm man pages. comppath : str, optional @@ -153,7 +170,7 @@ def main(fp=sys.stderr, argv=None): if argv is None: argv = sys.argv[1:] try: - log = argv.index('--log') + log_idx = argv.index('--log') except ValueError: for i in argv: if i.startswith('--log='): @@ -162,13 +179,12 @@ def main(fp=sys.stderr, argv=None): else: logLevel = 'INFO' else: - # argv.pop(log) - # logLevel = argv.pop(log) - logLevel = argv[log + 1] + # argv.pop(log_idx) + # logLevel = argv.pop(log_idx) + logLevel = argv[log_idx + 1] logging.basicConfig( level=getattr(logging, logLevel), format="%(levelname)s:%(module)s:%(lineno)d:%(message)s") - log = logging.getLogger(__name__) d = tqdm.__init__.__doc__ + CLI_EXTRA_DOC @@ -183,16 +199,17 @@ def main(fp=sys.stderr, argv=None): # d = RE_OPTS.sub(r' --\1=<\1> : \2', d) split = RE_OPTS.split(d) opt_types_desc = zip(split[1::3], split[2::3], split[3::3]) - d = ''.join('\n --{0}=<{0}> : {1}{2}'.format(*otd) + d = ''.join(('\n --{0} : {2}{3}' if otd[1] == 'bool' else + '\n --{0}=<{1}> : {2}{3}').format( + otd[0].replace('_', '-'), otd[0], *otd[1:]) for otd in opt_types_desc if otd[0] not in UNSUPPORTED_OPTS) d = """Usage: tqdm [--help | options] Options: - -h, --help Print this help and exit - -v, --version Print version and exit - + -h, --help Print this help and exit. + -v, --version Print version and exit. """ + d.strip('\n') + '\n' # opts = docopt(d, version=__version__) @@ -212,11 +229,19 @@ def main(fp=sys.stderr, argv=None): tqdm_args = {'file': fp} try: for (o, v) in opts.items(): + o = o.replace('-', '_') try: tqdm_args[o] = cast(v, opt_types[o]) except KeyError as e: raise TqdmKeyError(str(e)) log.debug('args:' + str(tqdm_args)) + + delim_per_char = tqdm_args.pop('bytes', False) + update = tqdm_args.pop('update', False) + update_to = tqdm_args.pop('update_to', False) + if sum((delim_per_char, update, update_to)) > 1: + raise TqdmKeyError( + "Can only have one of --bytes --update --update_to") except: fp.write('\nError:\nUsage:\n tqdm [--help | options]\n') for i in sys.stdin: @@ -225,12 +250,18 @@ def main(fp=sys.stderr, argv=None): else: buf_size = tqdm_args.pop('buf_size', 256) delim = tqdm_args.pop('delim', b'\\n') - delim_per_char = tqdm_args.pop('bytes', False) tee = tqdm_args.pop('tee', False) manpath = tqdm_args.pop('manpath', None) comppath = tqdm_args.pop('comppath', None) + if tqdm_args.pop('null', False): + class stdout(object): + @staticmethod + def write(_): + pass + else: + stdout = sys.stdout + stdout = getattr(stdout, 'buffer', stdout) stdin = getattr(sys.stdin, 'buffer', sys.stdin) - stdout = getattr(sys.stdout, 'buffer', sys.stdout) if manpath or comppath: from os import path from shutil import copyfile @@ -267,9 +298,32 @@ def write(x): posix_pipe(stdin, stdout, '', buf_size, t.update) elif delim == b'\\n': log.debug(tqdm_args) - for i in tqdm(stdin, **tqdm_args): - stdout.write(i) + if update or update_to: + with tqdm(**tqdm_args) as t: + if update: + def callback(i): + t.update(numeric(i)) + else: # update_to + def callback(i): + t.update(numeric(i) - t.n) + for i in stdin: + stdout.write(i) + callback(i) + else: + for i in tqdm(stdin, **tqdm_args): + stdout.write(i) else: log.debug(tqdm_args) with tqdm(**tqdm_args) as t: - posix_pipe(stdin, stdout, delim, buf_size, t.update) + callback_len = False + if update: + def callback(i): + t.update(numeric(i)) + elif update_to: + def callback(i): + t.update(numeric(i) - t.n) + else: + callback = t.update + callback_len = True + posix_pipe(stdin, stdout, delim, buf_size, + callback, callback_len) diff --git a/tqdm/completion.sh b/tqdm/completion.sh index c6304ab73..42e84a8b5 100755 --- a/tqdm/completion.sh +++ b/tqdm/completion.sh @@ -12,7 +12,7 @@ _tqdm(){ COMPREPLY=($(compgen -W 'CRITICAL FATAL ERROR WARN WARNING INFO DEBUG NOTSET' -- ${cur})) ;; *) - COMPREPLY=($(compgen -W '--ascii --bar_format --buf_size --bytes --comppath --delim --desc --disable --dynamic_ncols --help --initial --leave --lock_args --log --manpath --maxinterval --mininterval --miniters --ncols --nrows --position --postfix --smoothing --tee --total --unit --unit_divisor --unit_scale --version --write_bytes -h -v' -- ${cur})) + COMPREPLY=($(compgen -W '--ascii --bar_format --buf_size --bytes --comppath --delim --desc --disable --dynamic_ncols --help --initial --leave --lock_args --log --manpath --maxinterval --mininterval --miniters --ncols --nrows --null --position --postfix --smoothing --tee --total --unit --unit_divisor --unit_scale --update --update_to --version --write_bytes -h -v' -- ${cur})) ;; esac } diff --git a/tqdm/tests/tests_main.py b/tqdm/tests/tests_main.py index dbbf3c16a..08eccc05b 100644 --- a/tqdm/tests/tests_main.py +++ b/tqdm/tests/tests_main.py @@ -47,16 +47,17 @@ def test_pipes(): def test_main(): """Test misc CLI options""" _SYS = sys.stdin, sys.argv + N = 123 # test direct import - sys.stdin = map(str, _range(int(123))) + sys.stdin = map(str, _range(N)) sys.argv = ['', '--desc', 'Test CLI import', '--ascii', 'True', '--unit_scale', 'True'] import tqdm.__main__ # NOQA sys.stderr.write("Test misc CLI options ... ") # test --delim - IN_DATA = '\0'.join(map(str, _range(int(123)))) + IN_DATA = '\0'.join(map(str, _range(N))) with closing(StringIO()) as sys.stdin: sys.argv = ['', '--desc', 'Test CLI delim', '--ascii', 'True', '--delim', r'\0', '--buf_size', '64'] @@ -65,7 +66,7 @@ def test_main(): sys.stdin.seek(0) with closing(UnicodeIO()) as fp: main(fp=fp) - assert "123it" in fp.getvalue() + assert str(N) + "it" in fp.getvalue() # test --bytes IN_DATA = IN_DATA.replace('\0', '\n') @@ -78,7 +79,7 @@ def test_main(): assert str(len(IN_DATA)) in fp.getvalue() # test --log - sys.stdin = map(str, _range(int(123))) + sys.stdin = map(str, _range(N)) # with closing(UnicodeIO()) as fp: main(argv=['--log', 'DEBUG'], fp=NULL) # assert "DEBUG:" in sys.stdout.getvalue() @@ -99,6 +100,85 @@ def test_main(): # spaces to clear intermediate lines could increase length assert len(fp.getvalue()) >= res + len(IN_DATA) + # test --null + _STDOUT = sys.stdout + try: + with closing(StringIO()) as sys.stdout: + with closing(StringIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + + sys.stdin.seek(0) + with closing(UnicodeIO()) as fp: + main(argv=['--null'], fp=fp) + assert not sys.stdout.getvalue() + + with closing(StringIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + + sys.stdin.seek(0) + with closing(UnicodeIO()) as fp: + main(argv=[], fp=fp) + assert sys.stdout.getvalue() + except: + sys.stdout = _STDOUT + raise + else: + sys.stdout = _STDOUT + + # test integer --update + with closing(StringIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + + sys.stdin.seek(0) + with closing(UnicodeIO()) as fp: + main(argv=['--update'], fp=fp) + res = fp.getvalue() + assert str(N // 2 * N) + "it" in res # arithmetic sum formula + + # test integer --update --delim + with closing(StringIO()) as sys.stdin: + sys.stdin.write(IN_DATA.replace('\n', 'D')) + + sys.stdin.seek(0) + with closing(UnicodeIO()) as fp: + main(argv=['--update', '--delim', 'D'], fp=fp) + res = fp.getvalue() + assert str(N // 2 * N) + "it" in res # arithmetic sum formula + + # test integer --update_to + with closing(StringIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + + sys.stdin.seek(0) + with closing(UnicodeIO()) as fp: + main(argv=['--update-to'], fp=fp) + res = fp.getvalue() + assert str(N - 1) + "it" in res + assert str(N) + "it" not in res + + # test integer --update_to --delim + with closing(StringIO()) as sys.stdin: + sys.stdin.write(IN_DATA.replace('\n', 'D')) + + sys.stdin.seek(0) + with closing(UnicodeIO()) as fp: + main(argv=['--update-to', '--delim', 'D'], fp=fp) + res = fp.getvalue() + assert str(N - 1) + "it" in res + assert str(N) + "it" not in res + + # test float --update_to + IN_DATA = '\n'.join((str(i / 2.0) for i in _range(N))) + with closing(StringIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + + sys.stdin.seek(0) + with closing(UnicodeIO()) as fp: + main(argv=['--update-to'], fp=fp) + res = fp.getvalue() + assert str((N - 1) / 2.0) + "it" in res + assert str(N / 2.0) + "it" not in res + # clean up sys.stdin, sys.argv = _SYS @@ -150,7 +230,7 @@ def test_comppath(): def test_exceptions(): """Test CLI Exceptions""" _SYS = sys.stdin, sys.argv - sys.stdin = map(str, _range(int(123))) + sys.stdin = map(str, _range(123)) sys.argv = ['', '-ascii', '-unit_scale', '--bad_arg_u_ment', 'foo'] try: @@ -179,6 +259,15 @@ def test_exceptions(): else: raise TqdmTypeError('invalid_int_value') + sys.argv = ['', '--update', '--update_to'] + try: + main(fp=NULL) + except TqdmKeyError as e: + if 'Can only have one of --' not in str(e): + raise + else: + raise TqdmKeyError('Cannot have both --update --update_to') + # test SystemExits for i in ('-h', '--help', '-v', '--version'): sys.argv = ['', i] diff --git a/tqdm/tqdm.1 b/tqdm/tqdm.1 index c983d20d1..e8dadade2 100644 --- a/tqdm/tqdm.1 +++ b/tqdm/tqdm.1 @@ -36,12 +36,12 @@ counting:\ 100%|█████████|\ 432/432\ [00:00<00:00,\ 794361.83f .SH OPTIONS .TP .B \-h, \-\-help -Print this help and exit +Print this help and exit. .RS .RE .TP .B \-v, \-\-version -Print version and exit +Print version and exit. .RS .RE .TP @@ -63,7 +63,7 @@ specify an initial arbitrary large positive number, e.g. .RS .RE .TP -.B \-\-leave=\f[I]leave\f[] +.B \-\-leave bool, optional. If [default: True], keeps all traces of the progressbar upon termination of iteration. @@ -117,7 +117,7 @@ The fallback is to use ASCII characters " 123456789#". .RS .RE .TP -.B \-\-disable=\f[I]disable\f[] +.B \-\-disable bool, optional. Whether to disable the entire progressbar wrapper [default: False]. If set to None, disable on non\-TTY. @@ -131,7 +131,7 @@ it]. .RS .RE .TP -.B \-\-unit_scale=\f[I]unit_scale\f[] +.B \-\-unit\-scale=\f[I]unit_scale\f[] bool or int or float, optional. If 1 or True, the number of iterations will be reduced/scaled automatically and a metric prefix following the International System of @@ -140,7 +140,7 @@ If any other non\-zero number, will scale \f[C]total\f[] and \f[C]n\f[]. .RS .RE .TP -.B \-\-dynamic_ncols=\f[I]dynamic_ncols\f[] +.B \-\-dynamic\-ncols bool, optional. If set, constantly alters \f[C]ncols\f[] and \f[C]nrows\f[] to the environment (allowing for window resizes) [default: False]. @@ -156,7 +156,7 @@ Ranges from 0 (average speed) to 1 (current/instantaneous speed) .RS .RE .TP -.B \-\-bar_format=\f[I]bar_format\f[] +.B \-\-bar\-format=\f[I]bar_format\f[] str, optional. Specify a custom bar string formatting. May impact performance. @@ -196,13 +196,13 @@ Calls \f[C]set_postfix(**postfix)\f[] if possible (dict). .RS .RE .TP -.B \-\-unit_divisor=\f[I]unit_divisor\f[] +.B \-\-unit\-divisor=\f[I]unit_divisor\f[] float, optional. [default: 1000], ignored unless \f[C]unit_scale\f[] is True. .RS .RE .TP -.B \-\-write_bytes=\f[I]write_bytes\f[] +.B \-\-write\-bytes bool, optional. If (default: None) and \f[C]file\f[] is unspecified, bytes will be written in Python 2. @@ -211,7 +211,7 @@ In all other cases will default to unicode. .RS .RE .TP -.B \-\-lock_args=\f[I]lock_args\f[] +.B \-\-lock\-args=\f[I]lock_args\f[] tuple, optional. Passed to \f[C]refresh\f[] for intermediate output (initialisation, iterating, and updating). @@ -236,14 +236,14 @@ N.B.: on Windows systems, Python converts \[aq]\\n\[aq] to .RS .RE .TP -.B \-\-buf_size=\f[I]buf_size\f[] +.B \-\-buf\-size=\f[I]buf_size\f[] int, optional. String buffer size in bytes [default: 256] used when \f[C]delim\f[] is specified. .RS .RE .TP -.B \-\-bytes=\f[I]bytes\f[] +.B \-\-bytes bool, optional. If true, will count bytes, ignore \f[C]delim\f[], and default \f[C]unit_scale\f[] to True, \f[C]unit_divisor\f[] to 1024, and @@ -251,13 +251,33 @@ If true, will count bytes, ignore \f[C]delim\f[], and default .RS .RE .TP -.B \-\-tee=\f[I]tee\f[] +.B \-\-tee bool, optional. If true, passes \f[C]stdin\f[] to both \f[C]stderr\f[] and \f[C]stdout\f[]. .RS .RE .TP +.B \-\-update +bool, optional. +If true, will treat input as newly elapsed iterations, i.e. +numbers to pass to \f[C]update()\f[]. +.RS +.RE +.TP +.B \-\-update\-to +bool, optional. +If true, will treat input as total elapsed iterations, i.e. +numbers to assign to \f[C]self.n\f[]. +.RS +.RE +.TP +.B \-\-null +bool, optional. +If true, will discard input (no stdout). +.RS +.RE +.TP .B \-\-manpath=\f[I]manpath\f[] str, optional. Directory in which to install tqdm man pages.