From 46c23cd50ec005b06b56f6f89892852ad4a44855 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 02:05:11 -0500 Subject: [PATCH 01/15] Provide a way to capture or wrap streams - Also fix a bug with error reporting during `_create_subprocess` failures - Adds a stream wrapper using `TextIOWrapper` via `vistir.misc.StreamWrapper` - Adds `vistir.misc.get_wrapped_stream()` function to wrap existing streams - Adds `vistir.contextmanagers.replaced_stream()` to temporarily replace a stream - Fixes #49 - Closes #48 Signed-off-by: Dan Ryan --- README.rst | 121 +++++++++++++++++++++++++++++++++- docs/quickstart.rst | 120 +++++++++++++++++++++++++++++++++ news/48.feature.rst | 3 + news/49.bugfix.rst | 1 + setup.cfg | 2 +- src/vistir/contextmanagers.py | 40 ++++++++++- src/vistir/misc.py | 82 ++++++++++++++++++++++- src/vistir/path.py | 1 + 8 files changed, 365 insertions(+), 5 deletions(-) create mode 100644 news/48.feature.rst create mode 100644 news/49.bugfix.rst diff --git a/README.rst b/README.rst index 198f354..33cf53c 100644 --- a/README.rst +++ b/README.rst @@ -131,6 +131,7 @@ default encoding: * ``vistir.contextmanagers.atomic_open_for_write`` * ``vistir.contextmanagers.cd`` * ``vistir.contextmanagers.open_file`` + * ``vistir.contextmanagers.replaced_stream`` * ``vistir.contextmanagers.spinner`` * ``vistir.contextmanagers.temp_environ`` * ``vistir.contextmanagers.temp_path`` @@ -203,6 +204,23 @@ to pair this with an iterator which employs a sensible chunk size. shutil.copyfileobj(fp, filecontents) +.. _`replaced_stream`: + +A context manager to temporarily swap out *stream_name* with a stream wrapper. This will +capture the stream output and prevent it from being written as normal. + +.. code-block:: python + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + ... assert orig_stdout.getvalue() != "hello" + + >>> sys.stdout.write("hello") + 'hello' + + .. _`spinner`: **spinner** @@ -286,8 +304,13 @@ The following Miscellaneous utilities are available as helper methods: * ``vistir.misc.partialclass`` * ``vistir.misc.to_text`` * ``vistir.misc.to_bytes`` + * ``vistir.misc.divide`` + * ``vistir.misc.take`` + * ``vistir.misc.chunked`` * ``vistir.misc.decode_for_output`` - + * ``vistir.misc.get_canonical_encoding_name`` + * ``vistir.misc.get_wrapped_stream`` + * ``vistir.misc.StreamWrapper`` .. _`shell_escape`: @@ -401,6 +424,62 @@ Converts arbitrary byte-convertable input to bytes while handling errors. b'this is some text' +.. _`chunked`: + +**chunked** +//////////// + +Splits an iterable up into groups *of the specified length*, per `more itertools`_. Returns an iterable. + +This example will create groups of chunk size **5**, which means there will be *6 groups*. + +.. code-block:: python + + >>> chunked_iterable = vistir.misc.chunked(5, range(30)) + >>> for chunk in chunked_iterable: + ... add_to_some_queue(chunk) + +.. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + +.. _`take`: + +**take** +///////// + +Take elements from the supplied iterable without consuming it. + +.. code-block:: python + + >>> iterable = range(30) + >>> first_10 = take(10, iterable) + >>> [i for i in first_10] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> [i for i in iterable] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + + +.. _`divide`: + +**divide** +//////////// + +Splits an iterable up into the *specified number of groups*, per `more itertools`_. Returns an iterable. + +.. code-block:: python + + >>> iterable = range(30) + >>> groups = [] + >>> for grp in vistir.misc.divide(3, iterable): + ... groups.append(grp) + >>> groups + [, , ] + + +.. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + .. _`decode_for_output`: **decode_for_output** @@ -411,6 +490,46 @@ outputs using the system preferred locale using ``locale.getpreferredencoding(Fa with some additional hackery on linux systems. +.. _`get_canonical_encoding_name`: + +**get_canonical_encoding_name** +//////////////////////////////// + +Given an encoding name, get the canonical name from a codec lookup. + +.. code-block:: python + + >>> vistir.misc.get_canonical_encoding_name("utf8") + "utf-8" + + +.. _`get_wrapped_stream`: + +**get_wrapped_stream** +////////////////////// + +Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + +.. code-block:: python + + >>> stream = sys.stdout + >>> wrapped_stream = vistir.misc.get_wrapped_stream(sys.stdout) + + +.. _`StreamWrapper`: + +**StreamWrapper** +////////////////// + +A stream wrapper and compatibility class for handling wrapping file-like stream objects +which may be used in place of ``sys.stdout`` and other streams. + +.. code-block:: python + + >>> wrapped_stream = vistir.misc.StreamWrapper(sys.stdout, encoding="utf-8", errors="replace", line_buffering=True) + >>> wrapped_stream = vistir.misc.StreamWrapper(io.StringIO(), encoding="utf-8", errors="replace", line_buffering=True) + + 🐉 Path Utilities ------------------ diff --git a/docs/quickstart.rst b/docs/quickstart.rst index b9d0102..3b93b0f 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -131,6 +131,7 @@ defualt encoding: * :func:`~vistir.contextmanagers.atomic_open_for_write` * :func:`~vistir.contextmanagers.cd` * :func:`~vistir.contextmanagers.open_file` + * :func:`~vistir.contextmanagers.replaced_stream` * :func:`~vistir.contextmanagers.spinner` * :func:`~vistir.contextmanagers.temp_environ` * :func:`~vistir.contextmanagers.temp_path` @@ -203,6 +204,23 @@ to pair this with an iterator which employs a sensible chunk size. shutil.copyfileobj(fp, filecontents) +.. _`replaced_stream`: + +A context manager to temporarily swap out *stream_name* with a stream wrapper. This will +capture the stream output and prevent it from being written as normal. + +.. code-block:: python + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + ... assert orig_stdout.getvalue() != "hello" + + >>> sys.stdout.write("hello") + 'hello' + + .. _`spinner`: **spinner** @@ -286,7 +304,13 @@ The following Miscellaneous utilities are available as helper methods: * :func:`~vistir.misc.partialclass` * :func:`~vistir.misc.to_text` * :func:`~vistir.misc.to_bytes` + * :func:`~vistir.misc.divide` + * :func:`~vistir.misc.take` + * :func:`~vistir.misc.chunked` * :func:`~vistir.misc.decode_for_output` + * :func:`~vistir.misc.get_canonical_encoding_name` + * :func:`~vistir.misc.get_wrapped_stream` + * :class:`~vistir.misc.StreamWrapper` .. _`shell_escape`: @@ -401,6 +425,62 @@ Converts arbitrary byte-convertable input to bytes while handling errors. b'this is some text' +.. _`chunked`: + +**chunked** +//////////// + +Splits an iterable up into groups *of the specified length*, per `more itertools`_. Returns an iterable. + +This example will create groups of chunk size **5**, which means there will be *6 groups*. + +.. code-block:: python + + >>> chunked_iterable = vistir.misc.chunked(5, range(30)) + >>> for chunk in chunked_iterable: + ... add_to_some_queue(chunk) + +.. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + +.. _`take`: + +**take** +///////// + +Take elements from the supplied iterable without consuming it. + +.. code-block:: python + + >>> iterable = range(30) + >>> first_10 = take(10, iterable) + >>> [i for i in first_10] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> [i for i in iterable] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + + +.. _`divide`: + +**divide** +//////////// + +Splits an iterable up into the *specified number of groups*, per `more itertools`_. Returns an iterable. + +.. code-block:: python + + >>> iterable = range(30) + >>> groups = [] + >>> for grp in vistir.misc.divide(3, iterable): + ... groups.append(grp) + >>> groups + [, , ] + + +.. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + .. _`decode_for_output`: **decode_for_output** @@ -416,6 +496,46 @@ with some additional hackery on linux systems. "some default locale encoded text" +.. _`get_canonical_encoding_name`: + +**get_canonical_encoding_name** +//////////////////////////////// + +Given an encoding name, get the canonical name from a codec lookup. + +.. code-block:: python + + >>> vistir.misc.get_canonical_encoding_name("utf8") + "utf-8" + + +.. _`get_wrapped_stream`: + +**get_wrapped_stream** +////////////////////// + +Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + +.. code-block:: python + + >>> stream = sys.stdout + >>> wrapped_stream = vistir.misc.get_wrapped_stream(sys.stdout) + + +.. _`StreamWrapper`: + +**StreamWrapper** +////////////////// + +A stream wrapper and compatibility class for handling wrapping file-like stream objects +which may be used in place of ``sys.stdout`` and other streams. + +.. code-block:: python + + >>> wrapped_stream = vistir.misc.StreamWrapper(sys.stdout, encoding="utf-8", errors="replace", line_buffering=True) + >>> wrapped_stream = vistir.misc.StreamWrapper(io.StringIO(), encoding="utf-8", errors="replace", line_buffering=True) + + 🐉 Path Utilities ------------------ diff --git a/news/48.feature.rst b/news/48.feature.rst new file mode 100644 index 0000000..92cf00e --- /dev/null +++ b/news/48.feature.rst @@ -0,0 +1,3 @@ +Added a new ``vistir.misc.StreamWrapper`` class with ``vistir.misc.get_wrapped_stream()`` to wrap existing streams +and ``vistir.contextmanagers.replaced_stream()`` to temporarily replace a stream. + diff --git a/news/49.bugfix.rst b/news/49.bugfix.rst new file mode 100644 index 0000000..a8c2be7 --- /dev/null +++ b/news/49.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug with exception handling during ``_create_process`` calls. diff --git a/setup.cfg b/setup.cfg index 55026d5..9622e37 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,7 +34,7 @@ python_requires = >=2.6,!=3.0,!=3.1,!=3.2,!=3.3 setup_requires = setuptools>=36.2.2 install_requires = pathlib2;python_version<"3.5" - backports.functools_lru_cache;python_version <= "3.4" + backports.functools_lru_cache;python_version<="3.4" backports.shutil_get_terminal_size;python_version<"3.3" backports.weakref;python_version<"3.3" requests diff --git a/src/vistir/contextmanagers.py b/src/vistir/contextmanagers.py index 77fbb9d..aebd37d 100644 --- a/src/vistir/contextmanagers.py +++ b/src/vistir/contextmanagers.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function import io import os @@ -13,9 +13,14 @@ from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path +if six.PY2: + from io import BytesIO as StringIO +else: + from io import StringIO + __all__ = [ - "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" + "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner", "dummy_spinner", ] @@ -286,3 +291,34 @@ def open_file(link, session=None, stream=True): if conn is not None: conn.close() result.close() + + +@contextmanager +def replaced_stream(stream_name): + """ + Context manager to temporarily swap out *stream_name* with a stream wrapper. + + :param str stream_name: The name of a sys stream to wrap + :returns: A ``StreamWrapper`` replacement, temporarily + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + ... assert orig_stdout.getvalue() != "hello" + + >>> sys.stdout.write("hello") + 'hello' + """ + from .misc import StreamWrapper, get_canonical_encoding_name, PREFERRED_ENCODING + orig_stream = getattr(sys, stream_name) + encoding = get_canonical_encoding_name( + getattr(orig_stream, encoding, PREFERRED_ENCODING) + ) + new_stream = StringIO() + wrapped_stream = StreamWrapper(new_stream, encoding, "replace", line_buffering=True) + try: + setattr(sys, stream_name, wrapped_stream) + yield getattr(sys, stream_name) + finally: + setattr(sys, stream_name, orig_stream) diff --git a/src/vistir/misc.py b/src/vistir/misc.py index 8537be7..f107f26 100644 --- a/src/vistir/misc.py +++ b/src/vistir/misc.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import io import json import logging import locale @@ -38,6 +39,9 @@ class WindowsError(OSError): "divide", "getpreferredencoding", "decode_for_output", + "get_canonical_encoding_name", + "get_wrapped_stream", + "StreamWrapper", ] @@ -159,7 +163,7 @@ def _create_subprocess( c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr) except Exception as exc: - sys.stderr.write("Error %s while executing command %s", exc, " ".join(cmd._parts)) + sys.stderr.write("Error %s while executing command %s", % (exc, " ".join(cmd._parts))) raise if not block: c.stdin.close() @@ -530,3 +534,79 @@ def decode_for_output(output): pass output = output.decode(PREFERRED_ENCODING) return output + + +def get_canonical_encoding_name(name): + # type: (str) -> str + """ + Given an encoding name, get the canonical name from a codec lookup. + + :param str name: The name of the codec to lookup + :return: The canonical version of the codec name + :rtype: str + """ + + import codecs + try: + codec = codecs.lookup(name) + except LookupError: + return name + else: + return codec.name + + +def get_wrapped_stream(stream): + """ + Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + + :param stream: A stream instance to wrap + :returns: A new, wrapped stream + :rtype: :class:`StreamWrapper` + """ + + if stream is None: + raise TypeError("must provide a stream to wrap") + encoding = getattr(stream, encoding, PREFERRED_ENCODING) + encoding = get_canonical_encoding_name(encoding) + return StreamWrapper(stream, encoding, "replace", line_buffering=True) + + +class StreamWrapper(io.TextIOWrapper): + + """ + This wrapper class will wrap a provided stream and supply an interface + for compatibility. + """ + + def __init__(self, stream, encoding, errors, line_buffering=True, **kwargs): + self._stream = stream + self.errors = errors + self.line_buffering = line_buffering + io.TextIOWrapper.__init__( + self, stream, encoding, errors, line_buffering=line_buffering, **kwargs + ) + + # borrowed from click's implementation of stream wrappers, see + # https://github.com/pallets/click/blob/6cafd32/click/_compat.py#L64 + if six.PY2: + def write(self, x): + if isinstance(x, (str, buffer, bytearray)): + try: + self.flush() + except Exception: + pass + return self.buffer.write(str(x)) + return io.TextIOWrapper.write(self, x) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __del__(self): + try: + self.detach() + except Exception: + pass + + def isatty(self): + return self._stream.isatty() diff --git a/src/vistir/path.py b/src/vistir/path.py index 91a9d66..6e9a7f6 100644 --- a/src/vistir/path.py +++ b/src/vistir/path.py @@ -31,6 +31,7 @@ "get_converted_relative_path", "handle_remove_readonly", "normalize_path", + "is_in_path", "is_file_url", "is_readonly_path", "is_valid_url", From b14e1440a785a196070eaff5a6fbd6d781402b69 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 02:27:00 -0500 Subject: [PATCH 02/15] Fix traceback formatting Signed-off-by: Dan Ryan --- src/vistir/misc.py | 5 ++++- src/vistir/spin.py | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/vistir/misc.py b/src/vistir/misc.py index f107f26..3b4e462 100644 --- a/src/vistir/misc.py +++ b/src/vistir/misc.py @@ -163,7 +163,10 @@ def _create_subprocess( c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr) except Exception as exc: - sys.stderr.write("Error %s while executing command %s", % (exc, " ".join(cmd._parts))) + import traceback + formatted_tb = "".join(traceback.format_exception(*sys.exc_info())) + sys.stderr.write("Error while executing command %s:" % " ".join(cmd._parts)) + sys.stderr.write(formatted_tb) raise if not block: c.stdin.close() diff --git a/src/vistir/spin.py b/src/vistir/spin.py index 7bdc85e..576cc51 100644 --- a/src/vistir/spin.py +++ b/src/vistir/spin.py @@ -54,7 +54,8 @@ def __exit__(self, exc_type, exc_val, traceback): if exc_type: import traceback from .misc import decode_for_output - self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) + formatted_tb = "".join(traceback.format_exception(*sys.exc_info())) + self.write_err(decode_for_output(formatted_tb)) self._close_output_buffer() return False @@ -181,7 +182,7 @@ def __init__(self, *args, **kwargs): self.out_buff = StringIO() self.write_to_stdout = write_to_stdout self.is_dummy = bool(yaspin is None) - if os.environ.get("ANSI_COLORS_DISABLED", False): + if DISABLE_COLORS: colorama.deinit() super(VistirSpinner, self).__init__(*args, **kwargs) From 259ba300cd5d0f5fb342dcd68dccd5c7802c8c85 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 02:27:53 -0500 Subject: [PATCH 03/15] add missing entry to __all__ Signed-off-by: Dan Ryan --- src/vistir/contextmanagers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vistir/contextmanagers.py b/src/vistir/contextmanagers.py index aebd37d..59c49de 100644 --- a/src/vistir/contextmanagers.py +++ b/src/vistir/contextmanagers.py @@ -20,7 +20,8 @@ __all__ = [ - "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner", "dummy_spinner", + "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner", + "dummy_spinner", "replaced_stream" ] From 0640f24d6a750ac5d18a3ec75c54f18f8a17366c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 14:29:24 -0500 Subject: [PATCH 04/15] Implementation fixes and tests Signed-off-by: Dan Ryan --- docs/conf.py | 31 ++++++++++--- docs/quickstart.rst | 1 - src/vistir/compat.py | 4 +- src/vistir/contextmanagers.py | 15 +------ src/vistir/misc.py | 85 +++++++++++++++++++++++++++++++---- src/vistir/spin.py | 29 +++++++++--- tests/test_contextmanagers.py | 8 ++++ tests/test_path.py | 1 + 8 files changed, 140 insertions(+), 34 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index df13239..8e3007b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,11 +12,30 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import codecs import os +import re import sys docs_dir = os.path.abspath(os.path.dirname(__file__)) src_dir = os.path.join(os.path.dirname(docs_dir), "src", "vistir") sys.path.insert(0, src_dir) +version_file = os.path.join(src_dir, "__init__.py") + + +def read_file(path): + # intentionally *not* adding an encoding option to open, See: + # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 + with codecs.open(path, 'r') as fp: + return fp.read() + + +def find_version(file_path): + version_file = read_file(file_path) + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + return '0.0.0' # -- Project information ----------------------------------------------------- @@ -25,10 +44,12 @@ copyright = '2018, Dan Ryan ' author = 'Dan Ryan ' +release = find_version(version_file) +version = '.'.join(release.split('.')[:2]) # The short X.Y version -version = '0.0' +# version = '0.0' # The full version, including alpha/beta/rc tags -release = '0.0.0.dev0' +# release = '0.0.0.dev0' # -- General configuration --------------------------------------------------- @@ -132,11 +153,11 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # - # 'papersize': 'letterpaper', + 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # - # 'pointsize': '10pt', + 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # @@ -173,7 +194,7 @@ # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'vistir', 'vistir Documentation', - author, 'vistir', 'One line description of project.', + author, 'vistir', 'Miscellaneous utilities for dealing with filesystems, paths, projects, subprocesses, and more.', 'Miscellaneous'), ] diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 3b93b0f..8560a7d 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -215,7 +215,6 @@ capture the stream output and prevent it from being written as normal. >>> with replaced_stream("stdout") as stdout: ... sys.stdout.write("hello") ... assert stdout.getvalue() == "hello" - ... assert orig_stdout.getvalue() != "hello" >>> sys.stdout.write("hello") 'hello' diff --git a/src/vistir/compat.py b/src/vistir/compat.py index 0f91855..7a216f3 100644 --- a/src/vistir/compat.py +++ b/src/vistir/compat.py @@ -60,6 +60,8 @@ if six.PY2: + from io import BytesIO as StringIO + class ResourceWarning(Warning): pass @@ -81,7 +83,7 @@ class IsADirectoryError(OSError): else: from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError - + from io import StringIO six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) from six.moves import Iterable diff --git a/src/vistir/contextmanagers.py b/src/vistir/contextmanagers.py index 59c49de..9e1090c 100644 --- a/src/vistir/contextmanagers.py +++ b/src/vistir/contextmanagers.py @@ -10,14 +10,9 @@ import six -from .compat import NamedTemporaryFile, Path +from .compat import NamedTemporaryFile, Path, StringIO from .path import is_file_url, is_valid_url, path_to_url, url_to_path -if six.PY2: - from io import BytesIO as StringIO -else: - from io import StringIO - __all__ = [ "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner", @@ -306,20 +301,14 @@ def replaced_stream(stream_name): >>> with replaced_stream("stdout") as stdout: ... sys.stdout.write("hello") ... assert stdout.getvalue() == "hello" - ... assert orig_stdout.getvalue() != "hello" >>> sys.stdout.write("hello") 'hello' """ - from .misc import StreamWrapper, get_canonical_encoding_name, PREFERRED_ENCODING orig_stream = getattr(sys, stream_name) - encoding = get_canonical_encoding_name( - getattr(orig_stream, encoding, PREFERRED_ENCODING) - ) new_stream = StringIO() - wrapped_stream = StreamWrapper(new_stream, encoding, "replace", line_buffering=True) try: - setattr(sys, stream_name, wrapped_stream) + setattr(sys, stream_name, new_stream) yield getattr(sys, stream_name) finally: setattr(sys, stream_name, orig_stream) diff --git a/src/vistir/misc.py b/src/vistir/misc.py index 3b4e462..6ee904c 100644 --- a/src/vistir/misc.py +++ b/src/vistir/misc.py @@ -16,7 +16,7 @@ import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod, to_native_string, Iterable +from .compat import Path, fs_str, partialmethod, to_native_string, Iterable, StringIO from .contextmanagers import spinner as spinner if os.name != "nt": @@ -521,21 +521,42 @@ def getpreferredencoding(): PREFERRED_ENCODING = getpreferredencoding() -def decode_for_output(output): +def get_output_encoding(source_encoding): + """ + Given a source encoding, determine the preferred output encoding. + + :param str source_encoding: The encoding of the source material. + :returns: The output encoding to decode to. + :rtype: str + """ + + if source_encoding is not None: + if get_canonical_encoding_name(source_encoding) == 'ascii': + return 'utf-8' + return get_canonical_encoding_name(source_encoding) + return get_canonical_encoding_name(PREFERRED_ENCODING) + + +def decode_for_output(output, target_stream=None): """Given a string, decode it for output to a terminal :param str output: A string to print to a terminal + :param target_stream: A stream to write to, we will encode to target this stream if possible. :return: A re-encoded string using the preferred encoding :rtype: str """ if not isinstance(output, six.string_types): return output + encoding = None + if target_stream is not None: + encoding = getattr(target_stream, "encoding", None) + encoding = get_output_encoding(encoding) try: - output = output.encode(PREFERRED_ENCODING) + output = output.encode(encoding) except AttributeError: pass - output = output.decode(PREFERRED_ENCODING) + output = output.decode(encoding) return output @@ -569,8 +590,8 @@ def get_wrapped_stream(stream): if stream is None: raise TypeError("must provide a stream to wrap") - encoding = getattr(stream, encoding, PREFERRED_ENCODING) - encoding = get_canonical_encoding_name(encoding) + encoding = getattr(stream, "encoding", None) + encoding = get_output_encoding(encoding) return StreamWrapper(stream, encoding, "replace", line_buffering=True) @@ -582,9 +603,7 @@ class StreamWrapper(io.TextIOWrapper): """ def __init__(self, stream, encoding, errors, line_buffering=True, **kwargs): - self._stream = stream - self.errors = errors - self.line_buffering = line_buffering + self._stream = stream = _StreamProvider(stream) io.TextIOWrapper.__init__( self, stream, encoding, errors, line_buffering=line_buffering, **kwargs ) @@ -613,3 +632,51 @@ def __del__(self): def isatty(self): return self._stream.isatty() + + +# More things borrowed from click, this is because we are using `TextIOWrapper` instead of +# just a normal StringIO +class _StreamProvider(object): + def __init__(self, stream): + self._stream = stream + + def __getattr__(self, name): + return getattr(self._stream, name) + + def read1(self, size): + fn = getattr(self._stream, "read1", None) + if fn is not None: + return fn(size) + if six.PY2: + return self._stream.readline(size) + return self._stream.read(size) + + def readable(self): + fn = getattr(self._stream, "readable", None) + if fn is not None: + return fn() + try: + self._stream.read(0) + except Exception: + return False + return True + + def writeable(self): + fn = getattr(self._stream, "writeable", None) + if fn is not None: + return fn() + try: + self._stream.write(b"") + except Exception: + return False + return True + + def seekable(self): + fn = getattr(self._stream, "seekable", None) + if fn is not None: + return fn() + try: + self._stream.seek(self._stream.tell()) + except Exception: + return False + return True diff --git a/src/vistir/spin.py b/src/vistir/spin.py index 576cc51..db10e74 100644 --- a/src/vistir/spin.py +++ b/src/vistir/spin.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import functools import os @@ -23,11 +24,29 @@ else: from yaspin.spinners import Spinners -handler = None -if yaspin and os.name == "nt": - handler = yaspin.signal_handlers.default_handler -elif yaspin and os.name != "nt": - handler = yaspin.signal_handlers.fancy_handler +if os.name == "nt": + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.fail() + spinner.stop() + sys.exit(0) + +else: + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.red.fail("✘") + spinner.stop() + sys.exit(0) CLEAR_LINE = chr(27) + "[K" diff --git a/tests/test_contextmanagers.py b/tests/test_contextmanagers.py index 25878d7..cc2a4d0 100644 --- a/tests/test_contextmanagers.py +++ b/tests/test_contextmanagers.py @@ -85,3 +85,11 @@ def test_open_file(tmpdir): for chunk in iter(lambda: fp.read(16384), b""): local_contents += chunk assert local_contents == filecontents.read() + + +def test_replace_stream(capsys): + with vistir.contextmanagers.replaced_stream("stdout") as stdout: + sys.stdout.write("hello") + assert stdout.getvalue() == "hello" + out, err = capsys.readouterr() + assert out.strip() != "hello" diff --git a/tests/test_path.py b/tests/test_path.py index 6b713cb..6be633b 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -167,6 +167,7 @@ def test_walk_up(tmpdir): assert results == expected[i] +@settings(deadline=500) def test_handle_remove_readonly(tmpdir): test_file = tmpdir.join("test_file.txt") test_file.write_text("a bunch of text", encoding="utf-8") From 1611baaa09e76e9d26de4ae9756248c84aa5b2f1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 14:50:41 -0500 Subject: [PATCH 05/15] Fix travis and appveyor? Signed-off-by: Dan Ryan --- .travis.yml | 6 +++--- appveyor.yml | 6 +++--- src/vistir/contextmanagers.py | 2 +- tests/test_path.py | 3 +-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9ea281a..4dd2f0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - "pipenv install --dev" - "pipenv run pip install --upgrade -e .[spinner,tests]" script: - - "pipenv run pytest -v -n auto tests/" + - "pipenv run pytest -v tests/" jobs: include: @@ -33,7 +33,7 @@ jobs: - stage: coverage python: "3.6" install: - - "pip install --upgrade pip pipenv pytest-cov pytest-xdist pytest-timeout pytest" + - "pip install --upgrade pip pipenv pytest-cov pytest-timeout pytest" - "pipenv install --dev" script: - - "pipenv run pytest -n auto --timeout 300 --cov=vistir --cov-report=term-missing --cov-report=xml --cov-report=html tests" + - "pipenv run pytest --timeout 300 --cov=vistir --cov-report=term-missing --cov-report=xml --cov-report=html tests" diff --git a/appveyor.yml b/appveyor.yml index d07c3d8..1f377ae 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,9 +13,9 @@ environment: install: - "SET PATH=%PYTHON%\\;%PYTHON%\\Scripts;%PATH%" - "python --version" - - "python -m pip install --upgrade pip pipenv" + - "python -m pip install --upgrade pip pipenv pytest pytest-timeout pytest-cov" - "python -m pipenv install --dev" - - "python -m pipenv run pip install -e .[tests,spinner]" + - "python -m pipenv run pip install --upgrade -e .[tests,spinner]" build: off @@ -24,4 +24,4 @@ test_script: - "subst T: %TEMP%" - "set TEMP=T:\\" - "set TMP=T:\\" - - "python -m pipenv run pytest -n auto -v tests" + - "python -m pipenv run pytest -ra tests" diff --git a/src/vistir/contextmanagers.py b/src/vistir/contextmanagers.py index 9e1090c..4b676ea 100644 --- a/src/vistir/contextmanagers.py +++ b/src/vistir/contextmanagers.py @@ -306,7 +306,7 @@ def replaced_stream(stream_name): 'hello' """ orig_stream = getattr(sys, stream_name) - new_stream = StringIO() + new_stream = six.StringIO() try: setattr(sys, stream_name, new_stream) yield getattr(sys, stream_name) diff --git a/tests/test_path.py b/tests/test_path.py index 6be633b..cac7769 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -32,7 +32,7 @@ def test_safe_expandvars(): @given(legal_path_chars(), legal_path_chars()) -@settings(suppress_health_check=(HealthCheck.filter_too_much,)) +@settings(suppress_health_check=(HealthCheck.filter_too_much,), deadline=500) def test_mkdir_p(base_dir, subdir): assume(not any((dir_name in ["", ".", "./", ".."] for dir_name in [base_dir, subdir]))) assume(not (os.path.relpath(subdir, start=base_dir) == ".")) @@ -167,7 +167,6 @@ def test_walk_up(tmpdir): assert results == expected[i] -@settings(deadline=500) def test_handle_remove_readonly(tmpdir): test_file = tmpdir.join("test_file.txt") test_file.write_text("a bunch of text", encoding="utf-8") From 97ffaa96a370a2b36c00ac8efdf76d30377a69ab Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 15:34:52 -0500 Subject: [PATCH 06/15] fix spinner Signed-off-by: Dan Ryan --- src/vistir/spin.py | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/vistir/spin.py b/src/vistir/spin.py index db10e74..65d046f 100644 --- a/src/vistir/spin.py +++ b/src/vistir/spin.py @@ -72,9 +72,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, traceback): if exc_type: import traceback - from .misc import decode_for_output - formatted_tb = "".join(traceback.format_exception(*sys.exc_info())) - self.write_err(decode_for_output(formatted_tb)) + formatted_tb = traceback.format_exception(exc_type, exc_val, traceback) + self.write_err("".join(formatted_tb)) self._close_output_buffer() return False @@ -99,9 +98,9 @@ def fail(self, exitcode=1, text="FAIL"): from .misc import decode_for_output if text and text != "None": if self.write_to_stdout: - self.write(decode_for_output(text)) + self.write(text) else: - self.write_err(decode_for_output(text)) + self.write_err(text) self._close_output_buffer() def ok(self, text="OK"): @@ -119,9 +118,9 @@ def hide_and_write(self, text, target=None): from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r")) + target.write(decode_for_output("\r", target_stream=target)) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_for_output("{0}\n".format(text), target_stream=target)) target.write(CLEAR_LINE) self._show_cursor(target=target) @@ -131,21 +130,32 @@ def write(self, text=None): from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stdout.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stdout.write(line) - self.stdout.write(CLEAR_LINE) + if not self.stdout.closed: + stdout = self.stdout + else: + stdout = sys.stdout + text = decode_for_output(text, target_stream=stdout) + stdout.write(decode_for_output("\r", target_stream=stdout)) + line = decode_for_output("{0}\n".format(text), target_stream=stdout) + stdout.write(line) + stdout.write(CLEAR_LINE) def write_err(self, text=None): from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stderr.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stderr.write(line) - self.stderr.write(CLEAR_LINE) + if not self.stderr.closed: + stderr = self.stderr + else: + if sys.stderr.closed: + print(text) + return + stderr = sys.stderr + text = decode_for_output(text, target_stream=stderr) + stderr.write(decode_for_output("\r", target_stream=stderr)) + line = decode_for_output("{0}\n".format(text), target_stream=stderr) + stderr.write(line) + stderr.write(CLEAR_LINE) @staticmethod def _hide_cursor(target=None): @@ -201,9 +211,9 @@ def __init__(self, *args, **kwargs): self.out_buff = StringIO() self.write_to_stdout = write_to_stdout self.is_dummy = bool(yaspin is None) + super(VistirSpinner, self).__init__(*args, **kwargs) if DISABLE_COLORS: colorama.deinit() - super(VistirSpinner, self).__init__(*args, **kwargs) def ok(self, text="OK", err=False): """Set Ok (success) finalizer to a spinner.""" From a740eacbafdf34cbb9624de6da637a806e11aca2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 17:04:11 -0500 Subject: [PATCH 07/15] Capture spinner output separately Signed-off-by: Dan Ryan --- src/vistir/compat.py | 20 ++++++++++++++------ src/vistir/spin.py | 25 +++++++++++++++---------- tests/test_spinner.py | 39 +++++++++++++++++++++++++-------------- 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/vistir/compat.py b/src/vistir/compat.py index 7a216f3..ec879b0 100644 --- a/src/vistir/compat.py +++ b/src/vistir/compat.py @@ -26,6 +26,11 @@ "TemporaryDirectory", "NamedTemporaryFile", "to_native_string", + "Iterable", + "Mapping", + "Sequence", + "Set", + "ItemsView" ] if sys.version_info >= (3, 5): @@ -46,17 +51,17 @@ try: from weakref import finalize except ImportError: - from backports.weakref import finalize + from backports.weakref import finalize # type: ignore try: from functools import partialmethod except Exception: - from .backports.functools import partialmethod + from .backports.functools import partialmethod # type: ignore try: from json import JSONDecodeError except ImportError: # Old Pythons. - JSONDecodeError = ValueError + JSONDecodeError = ValueError # type: ignore if six.PY2: @@ -85,9 +90,12 @@ class IsADirectoryError(OSError): from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError from io import StringIO -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable - +six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) # type: ignore +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) # type: ignore +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # type: ignore +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore +six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) # type: ignore +from six.moves import Iterable, Mapping, Sequence, Set, ItemsView # type: ignore # noqa if not sys.warnoptions: warnings.simplefilter("default", ResourceWarning) diff --git a/src/vistir/spin.py b/src/vistir/spin.py index 65d046f..a9b3fcb 100644 --- a/src/vistir/spin.py +++ b/src/vistir/spin.py @@ -21,8 +21,12 @@ except ImportError: yaspin = None Spinners = None + SpinBase = None else: - from yaspin.spinners import Spinners + import yaspin.spinners + import yaspin.core + Spinners = yaspin.spinners.Spinners + SpinBase = yaspin.core.Yaspin if os.name == "nt": def handler(signum, frame, spinner): @@ -53,7 +57,6 @@ def handler(signum, frame, spinner): class DummySpinner(object): def __init__(self, text="", **kwargs): - super(DummySpinner, self).__init__() if DISABLE_COLORS: colorama.init() from .misc import decode_for_output @@ -62,6 +65,7 @@ def __init__(self, text="", **kwargs): self.stderr = kwargs.get("stderr", sys.stderr) self.out_buff = StringIO() self.write_to_stdout = kwargs.get("write_to_stdout", False) + super(DummySpinner, self).__init__() def __enter__(self): if self.text and self.text != "None": @@ -69,10 +73,10 @@ def __enter__(self): self.write(self.text) return self - def __exit__(self, exc_type, exc_val, traceback): + def __exit__(self, exc_type, exc_val, tb): if exc_type: import traceback - formatted_tb = traceback.format_exception(exc_type, exc_val, traceback) + formatted_tb = traceback.format_exception(exc_type, exc_val, tb) self.write_err("".join(formatted_tb)) self._close_output_buffer() return False @@ -96,7 +100,7 @@ def _close_output_buffer(self): def fail(self, exitcode=1, text="FAIL"): from .misc import decode_for_output - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: self.write(text) else: @@ -104,11 +108,11 @@ def fail(self, exitcode=1, text="FAIL"): self._close_output_buffer() def ok(self, text="OK"): - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.stdout.write(self.text) + self.write(text) else: - self.stderr.write(self.text) + self.write_err(text) self._close_output_buffer() return 0 @@ -166,10 +170,11 @@ def _show_cursor(target=None): pass -base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner +if SpinBase is None: + SpinBase = DummySpinner -class VistirSpinner(base_obj): +class VistirSpinner(SpinBase): "A spinner class for handling spinners on windows and posix." def __init__(self, *args, **kwargs): diff --git a/tests/test_spinner.py b/tests/test_spinner.py index 0a15cd5..8609aff 100644 --- a/tests/test_spinner.py +++ b/tests/test_spinner.py @@ -1,19 +1,30 @@ # -*- coding=utf-8 -*- -import vistir.spin +import pytest +from vistir.contextmanagers import replaced_stream import time -def test_spinner(capsys): - with vistir.spin.create_spinner(spinner_name="bouncingBar", text="Running...", nospin=False) as spinner: - time.sleep(3) - spinner.ok("Ok!") - out, err = capsys.readouterr() - assert out.strip().endswith("Ok!") - assert err.strip() == "" - with vistir.spin.create_spinner(spinner_name="bouncingBar", text="Running...", nospin=False, write_to_stdout=False) as spinner: - time.sleep(3) - spinner.ok("Ok!") - out, err = capsys.readouterr() - assert err.strip().endswith("Ok!") - assert out.strip() == "" +@pytest.mark.parametrize("nospin, write_to_stdout", ( + (True, True), (True, False), + (False, True), (False, False) +)) +def test_spinner(capture_streams, monkeypatch, nospin, write_to_stdout): + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + with monkeypatch.context() as m: + # m.setenv("VISTIR_DISABLE_COLORS", "1") + import vistir.spin + with vistir.spin.create_spinner( + spinner_name="bouncingBar", text="Running...", nospin=nospin, + write_to_stdout=write_to_stdout + ) as spinner: + time.sleep(3) + spinner.ok("Ok!") + out = stdout.getvalue().strip(vistir.spin.CLEAR_LINE).strip() + err = stderr.getvalue().strip(vistir.spin.CLEAR_LINE).strip() + if write_to_stdout: + assert "Ok!" in out.strip().splitlines()[-1], out + assert err.strip() == "", err + else: + assert "Ok!" in err.strip().splitlines()[-1], err From 5aa27af74d4783e88dfaf2f4bed03a705112a68b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 17:06:58 -0500 Subject: [PATCH 08/15] Fix spinner capture Signed-off-by: Dan Ryan --- tests/test_spinner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spinner.py b/tests/test_spinner.py index 8609aff..531163a 100644 --- a/tests/test_spinner.py +++ b/tests/test_spinner.py @@ -9,7 +9,7 @@ (True, True), (True, False), (False, True), (False, False) )) -def test_spinner(capture_streams, monkeypatch, nospin, write_to_stdout): +def test_spinner(monkeypatch, nospin, write_to_stdout): with replaced_stream("stdout") as stdout: with replaced_stream("stderr") as stderr: with monkeypatch.context() as m: From efa29b2ccacfd043370f04240116583a9d796525 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 17:16:50 -0500 Subject: [PATCH 09/15] Update output decoding function Signed-off-by: Dan Ryan --- src/vistir/misc.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/vistir/misc.py b/src/vistir/misc.py index 6ee904c..2925a3c 100644 --- a/src/vistir/misc.py +++ b/src/vistir/misc.py @@ -537,11 +537,12 @@ def get_output_encoding(source_encoding): return get_canonical_encoding_name(PREFERRED_ENCODING) -def decode_for_output(output, target_stream=None): +def decode_for_output(output, target_stream=None, translation_map=None): """Given a string, decode it for output to a terminal :param str output: A string to print to a terminal :param target_stream: A stream to write to, we will encode to target this stream if possible. + :param dict translation_map: A mapping of unicode character ordinals to replacement strings. :return: A re-encoded string using the preferred encoding :rtype: str """ @@ -554,10 +555,18 @@ def decode_for_output(output, target_stream=None): encoding = get_output_encoding(encoding) try: output = output.encode(encoding) + except (UnicodeDecodeError, UnicodeEncodeError): + if translation_map is not None: + if six.PY2: + output = unicode.translate( + to_text(output, encoding=encoding), translation_map + ) + else: + output = output.translate(translation_map) + output = output.encode(encoding, errors="replace") except AttributeError: pass - output = output.decode(encoding) - return output + return to_text(output, encoding=encoding, errors="replace") def get_canonical_encoding_name(name): From 0d06f19956beaa670b1a5234dc6976b03f9311d8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 17:36:47 -0500 Subject: [PATCH 10/15] Fix output encodings Signed-off-by: Dan Ryan --- src/vistir/spin.py | 53 ++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/src/vistir/spin.py b/src/vistir/spin.py index a9b3fcb..d58a19f 100644 --- a/src/vistir/spin.py +++ b/src/vistir/spin.py @@ -253,26 +253,31 @@ def hide_and_write(self, text, target=None): def write(self, text): if not self.write_to_stdout: return self.write_err(text) - from .misc import to_text - sys.stdout.write("\r") - self.stdout.write(CLEAR_LINE) + from .misc import decode_for_output + stdout = self.stdout + if self.stdout.closed: + stdout = sys.stdout + stdout.write(decode_for_output("\r", target_stream=stdout)) + stdout.write(decode_for_output(CLEAR_LINE, target_stream=stdout)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) - self.stdout.write(text) - self.out_buff.write(to_text(text)) + text = decode_for_output("{0}\n".format(text), target_stream=stdout) + stdout.write(text) + self.out_buff.write(decode_for_output(text, target_stream=self.out_buff)) def write_err(self, text): """Write error text in the terminal without breaking the spinner.""" - from .misc import to_text - - self.stderr.write("\r") - self.stderr.write(CLEAR_LINE) + from .misc import decode_for_output + stderr = self.stderr + if self.stderr.closed: + stderr = sys.stderr + stderr.write(decode_for_output("\r", target_stream=stderr)) + stderr.write(decode_for_output(CLEAR_LINE, target_stream=stderr)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) + text = decode_for_output("{0}\n".format(text), target_stream=stderr) self.stderr.write(text) - self.out_buff.write(to_text(text)) + self.out_buff.write(decode_for_output(text, target_stream=self.out_buff)) def start(self): if self._sigmap: @@ -307,26 +312,23 @@ def stop(self): if target.isatty(): self._show_cursor(target=target) - if self.stderr and self.stderr != sys.stderr: - self.stderr.close() - if self.stdout and self.stdout != sys.stdout: - self.stdout.close() self.out_buff.close() def _freeze(self, final_text, err=False): """Stop spinner, compose last frame and 'freeze' it.""" + from .misc import decode_for_output if not final_text: final_text = "" - text = to_native_string(final_text) + target = self.stderr if err else self.stdout + if target.closed: + target = sys.stderr if err else sys.stdout + text = decode_for_output(final_text, target_stream=target) self._last_frame = self._compose_out(text, mode="last") # Should be stopped here, otherwise prints after # self._freeze call will mess up the spinner self.stop() - if err or not self.write_to_stdout: - self.stderr.write(self._last_frame) - else: - self.stdout.write(self._last_frame) + target.write(self._last_frame) def _compose_color_func(self): fn = functools.partial( @@ -339,20 +341,21 @@ def _compose_color_func(self): def _compose_out(self, frame, mode=None): # Ensure Unicode input + from .misc import decode_for_output - frame = to_native_string(frame) + frame = decode_for_output(frame) if self._text is None: self._text = "" - text = to_native_string(self._text) + text = decode_for_output(self._text) if self._color_func is not None: frame = self._color_func(frame) if self._side == "right": frame, text = text, frame # Mode if not mode: - out = to_native_string("\r{0} {1}".format(frame, text)) + out = decode_for_output("\r{0} {1}".format(frame, text)) else: - out = to_native_string("{0} {1}\n".format(frame, text)) + out = decode_for_output("{0} {1}\n".format(frame, text)) return out def _spin(self): From cbb3d261cc9aa74c13f94db2e4633344f733ec5d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 18:15:55 -0500 Subject: [PATCH 11/15] Properly escape non-ascii characters frm printing Signed-off-by: Dan Ryan --- src/vistir/misc.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/src/vistir/misc.py b/src/vistir/misc.py index 2925a3c..7bfef44 100644 --- a/src/vistir/misc.py +++ b/src/vistir/misc.py @@ -537,6 +537,26 @@ def get_output_encoding(source_encoding): return get_canonical_encoding_name(PREFERRED_ENCODING) +def _encode(output, encoding=None, errors=None, translation_map=None): + if encoding is None: + encoding = PREFERRED_ENCODING + try: + output = output.encode(encoding) + except (UnicodeDecodeError, UnicodeEncodeError): + if translation_map is not None: + if six.PY2: + output = unicode.translate( + to_text(output, encoding=encoding, errors=errors), translation_map + ) + else: + output = output.translate(translation_map) + else: + output = to_text(output, encoding=encoding, errors=errors) + except AttributeError: + pass + return output + + def decode_for_output(output, target_stream=None, translation_map=None): """Given a string, decode it for output to a terminal @@ -554,18 +574,10 @@ def decode_for_output(output, target_stream=None, translation_map=None): encoding = getattr(target_stream, "encoding", None) encoding = get_output_encoding(encoding) try: - output = output.encode(encoding) + output = _encode(output, encoding=encoding, translation_map=translation_map) except (UnicodeDecodeError, UnicodeEncodeError): - if translation_map is not None: - if six.PY2: - output = unicode.translate( - to_text(output, encoding=encoding), translation_map - ) - else: - output = output.translate(translation_map) - output = output.encode(encoding, errors="replace") - except AttributeError: - pass + output = _encode(output, encoding=encoding, errors="replace", + translation_map=translation_map) return to_text(output, encoding=encoding, errors="replace") @@ -648,6 +660,7 @@ def isatty(self): class _StreamProvider(object): def __init__(self, stream): self._stream = stream + super(_StreamProvider, self).__init__() def __getattr__(self, name): return getattr(self._stream, name) @@ -670,8 +683,8 @@ def readable(self): return False return True - def writeable(self): - fn = getattr(self._stream, "writeable", None) + def writable(self): + fn = getattr(self._stream, "writable", None) if fn is not None: return fn() try: From f6b9045dd6ee4f493dbb9d5901bf5db1fa50dcf8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 18:35:20 -0500 Subject: [PATCH 12/15] Fixed `decode_for_output` gracefulness - Added more entries to `six.moves` from `collections.abc` - Fixes #51 - Fixes #52 Signed-off-by: Dan Ryan --- news/51.feature.rst | 1 + news/52.feature.rst | 2 ++ src/vistir/spin.py | 67 ++++++++++++++++++++++----------------------- 3 files changed, 36 insertions(+), 34 deletions(-) create mode 100644 news/51.feature.rst create mode 100644 news/52.feature.rst diff --git a/news/51.feature.rst b/news/51.feature.rst new file mode 100644 index 0000000..b51bfea --- /dev/null +++ b/news/51.feature.rst @@ -0,0 +1 @@ +Added new entries in ``vistir.compat`` to support movements to ``collections.abc``: ``Mapping``, ``Sequence``, ``Set``, ``ItemsView``. diff --git a/news/52.feature.rst b/news/52.feature.rst new file mode 100644 index 0000000..ff2b556 --- /dev/null +++ b/news/52.feature.rst @@ -0,0 +1,2 @@ +Improved ``decode_for_output`` to handle decoding failures gracefully by moving to an ``replace`` strategy. +Now also allows a translation map to be provided to translate specific non-ascii characters when writing to outputs. diff --git a/src/vistir/spin.py b/src/vistir/spin.py index d58a19f..6a63cb7 100644 --- a/src/vistir/spin.py +++ b/src/vistir/spin.py @@ -14,6 +14,7 @@ from .compat import to_native_string from .termcolors import COLOR_MAP, COLORS, colored, DISABLE_COLORS +from .misc import decode_for_output from io import StringIO try: @@ -54,13 +55,20 @@ def handler(signum, frame, spinner): CLEAR_LINE = chr(27) + "[K" +TRANSLATION_MAP = { + 10004: u"OK", + 10008: u"x", +} + + +decode_output = functools.partial(decode_for_output, translation_map=TRANSLATION_MAP) + class DummySpinner(object): def __init__(self, text="", **kwargs): if DISABLE_COLORS: colorama.init() - from .misc import decode_for_output - self.text = to_native_string(decode_for_output(text)) if text else "" + self.text = to_native_string(decode_output(text)) if text else "" self.stdout = kwargs.get("stdout", sys.stdout) self.stderr = kwargs.get("stderr", sys.stderr) self.out_buff = StringIO() @@ -99,7 +107,6 @@ def _close_output_buffer(self): pass def fail(self, exitcode=1, text="FAIL"): - from .misc import decode_for_output if text is not None and text != "None": if self.write_to_stdout: self.write(text) @@ -119,33 +126,30 @@ def ok(self, text="OK"): def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r", target_stream=target)) + target.write(decode_output("\r", target_stream=target)) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text), target_stream=target)) + target.write(decode_output("{0}\n".format(text), target_stream=target)) target.write(CLEAR_LINE) self._show_cursor(target=target) def write(self, text=None): if not self.write_to_stdout: return self.write_err(text) - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass if not self.stdout.closed: stdout = self.stdout else: stdout = sys.stdout - text = decode_for_output(text, target_stream=stdout) - stdout.write(decode_for_output("\r", target_stream=stdout)) - line = decode_for_output("{0}\n".format(text), target_stream=stdout) + text = decode_output(text, target_stream=stdout) + stdout.write(decode_output("\r", target_stream=stdout)) + line = decode_output("{0}\n".format(text), target_stream=stdout) stdout.write(line) stdout.write(CLEAR_LINE) def write_err(self, text=None): - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass if not self.stderr.closed: @@ -155,9 +159,9 @@ def write_err(self, text=None): print(text) return stderr = sys.stderr - text = decode_for_output(text, target_stream=stderr) - stderr.write(decode_for_output("\r", target_stream=stderr)) - line = decode_for_output("{0}\n".format(text), target_stream=stderr) + text = decode_output(text, target_stream=stderr) + stderr.write(decode_output("\r", target_stream=stderr)) + line = decode_output("{0}\n".format(text), target_stream=stderr) stderr.write(line) stderr.write(CLEAR_LINE) @@ -241,43 +245,40 @@ def fail(self, text="FAIL", err=False): def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r")) + target.write(decode_output("\r")) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output("{0}\n".format(text))) target.write(CLEAR_LINE) self._show_cursor(target=target) def write(self, text): if not self.write_to_stdout: return self.write_err(text) - from .misc import decode_for_output stdout = self.stdout if self.stdout.closed: stdout = sys.stdout - stdout.write(decode_for_output("\r", target_stream=stdout)) - stdout.write(decode_for_output(CLEAR_LINE, target_stream=stdout)) + stdout.write(decode_output("\r", target_stream=stdout)) + stdout.write(decode_output(CLEAR_LINE, target_stream=stdout)) if text is None: text = "" - text = decode_for_output("{0}\n".format(text), target_stream=stdout) + text = decode_output("{0}\n".format(text), target_stream=stdout) stdout.write(text) - self.out_buff.write(decode_for_output(text, target_stream=self.out_buff)) + self.out_buff.write(decode_output(text, target_stream=self.out_buff)) def write_err(self, text): """Write error text in the terminal without breaking the spinner.""" - from .misc import decode_for_output stderr = self.stderr if self.stderr.closed: stderr = sys.stderr - stderr.write(decode_for_output("\r", target_stream=stderr)) - stderr.write(decode_for_output(CLEAR_LINE, target_stream=stderr)) + stderr.write(decode_output("\r", target_stream=stderr)) + stderr.write(decode_output(CLEAR_LINE, target_stream=stderr)) if text is None: text = "" - text = decode_for_output("{0}\n".format(text), target_stream=stderr) + text = decode_output("{0}\n".format(text), target_stream=stderr) self.stderr.write(text) - self.out_buff.write(decode_for_output(text, target_stream=self.out_buff)) + self.out_buff.write(decode_output(text, target_stream=self.out_buff)) def start(self): if self._sigmap: @@ -316,13 +317,12 @@ def stop(self): def _freeze(self, final_text, err=False): """Stop spinner, compose last frame and 'freeze' it.""" - from .misc import decode_for_output if not final_text: final_text = "" target = self.stderr if err else self.stdout if target.closed: target = sys.stderr if err else sys.stdout - text = decode_for_output(final_text, target_stream=target) + text = decode_output(final_text, target_stream=target) self._last_frame = self._compose_out(text, mode="last") # Should be stopped here, otherwise prints after @@ -341,21 +341,20 @@ def _compose_color_func(self): def _compose_out(self, frame, mode=None): # Ensure Unicode input - from .misc import decode_for_output - frame = decode_for_output(frame) + frame = decode_output(frame) if self._text is None: self._text = "" - text = decode_for_output(self._text) + text = decode_output(self._text) if self._color_func is not None: frame = self._color_func(frame) if self._side == "right": frame, text = text, frame # Mode if not mode: - out = decode_for_output("\r{0} {1}".format(frame, text)) + out = decode_output("\r{0} {1}".format(frame, text)) else: - out = decode_for_output("{0} {1}\n".format(frame, text)) + out = decode_output("{0} {1}\n".format(frame, text)) return out def _spin(self): From e92fea4a8ea8b9725015981f353927caa1401925 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 18:52:49 -0500 Subject: [PATCH 13/15] Cleanup and reorganize a bit Signed-off-by: Dan Ryan Add some extra compat utils Signed-off-by: Dan Ryan Add setup.cfg changes Signed-off-by: Dan Ryan uh, and init... Signed-off-by: Dan Ryan add fixture Signed-off-by: Dan Ryan --- setup.cfg | 10 +++++++--- src/vistir/__init__.py | 4 ++++ src/vistir/compat.py | 12 ++++++++++-- src/vistir/misc.py | 2 +- src/vistir/spin.py | 3 ++- tests/conftest.py | 8 ++++++++ 6 files changed, 32 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9622e37..9626be1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,18 +31,22 @@ classifier = [options] zip_safe = true python_requires = >=2.6,!=3.0,!=3.1,!=3.2,!=3.3 -setup_requires = setuptools>=36.2.2 +setup_requires = + setuptools>=38.2.5 + invoke + parver + wheel install_requires = - pathlib2;python_version<"3.5" + colorama backports.functools_lru_cache;python_version<="3.4" backports.shutil_get_terminal_size;python_version<"3.3" backports.weakref;python_version<"3.3" + pathlib2;python_version<"3.5" requests six [options.extras_require] spinner = - colorama cursor yaspin tests = diff --git a/src/vistir/__init__.py b/src/vistir/__init__.py index f54a309..ec428e1 100644 --- a/src/vistir/__init__.py +++ b/src/vistir/__init__.py @@ -61,4 +61,8 @@ "take", "chunked", "divide", + "StringIO", + "get_wrapped_stream", + "StreamWrapper", + "replaced_stream" ] diff --git a/src/vistir/compat.py b/src/vistir/compat.py index ec879b0..8f957ec 100644 --- a/src/vistir/compat.py +++ b/src/vistir/compat.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import errno import os @@ -86,8 +86,16 @@ class IsADirectoryError(OSError): """The command does not work on directories""" pass + class FileExistsError(OSError): + def __init__(self, *args, **kwargs): + self.errno = errno.EEXIST + super(FileExistsError, self).__init__(*args, **kwargs) + else: - from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError + from builtins import ( + ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError, + FileExistsError + ) from io import StringIO six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) # type: ignore diff --git a/src/vistir/misc.py b/src/vistir/misc.py index 7bfef44..eb91d79 100644 --- a/src/vistir/misc.py +++ b/src/vistir/misc.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function import io import json diff --git a/src/vistir/spin.py b/src/vistir/spin.py index 6a63cb7..a927505 100644 --- a/src/vistir/spin.py +++ b/src/vistir/spin.py @@ -9,7 +9,6 @@ import time import colorama -import cursor import six from .compat import to_native_string @@ -19,10 +18,12 @@ try: import yaspin + import cursor except ImportError: yaspin = None Spinners = None SpinBase = None + cursor = None else: import yaspin.spinners import yaspin.core diff --git a/tests/conftest.py b/tests/conftest.py index ec04db6..1556d2c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,2 +1,10 @@ # -*- coding=utf-8 -*- import pytest +from vistir.contextmanagers import replaced_stream + + +@pytest.fixture(scope="function") +def capture_streams(): + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + yield (stdout, stderr) From 6411138346d62ae16f932bd186d4ceb815862030 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 19:24:23 -0500 Subject: [PATCH 14/15] allow replacing both stdout and stderr simultaneously Signed-off-by: Dan Ryan --- src/vistir/contextmanagers.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/vistir/contextmanagers.py b/src/vistir/contextmanagers.py index 4b676ea..5e10e70 100644 --- a/src/vistir/contextmanagers.py +++ b/src/vistir/contextmanagers.py @@ -312,3 +312,31 @@ def replaced_stream(stream_name): yield getattr(sys, stream_name) finally: setattr(sys, stream_name, orig_stream) + + +@contextmanager +def replaced_streams(): + """ + Context manager to replace both ``sys.stdout`` and ``sys.stderr`` using + ``replaced_stream`` + + returns: *(stdout, stderr)* + + >>> import sys + >>> with vistir.contextmanagers.replaced_streams() as streams: + >>> stdout, stderr = streams + >>> sys.stderr.write("test") + >>> sys.stdout.write("hello") + >>> assert stdout.getvalue() == "hello" + >>> assert stderr.getvalue() == "test" + + >>> stdout.getvalue() + 'hello' + + >>> stderr.getvalue() + 'test' + """ + + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + yield (stdout, stderr) From 73801353c0d92922d573a1a4ecd57ad5f7dd5e67 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 9 Dec 2018 20:29:58 -0500 Subject: [PATCH 15/15] fix some issues Signed-off-by: Dan Ryan --- src/vistir/__init__.py | 4 ++++ src/vistir/contextmanagers.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/vistir/__init__.py b/src/vistir/__init__.py index ec428e1..b7fc190 100644 --- a/src/vistir/__init__.py +++ b/src/vistir/__init__.py @@ -6,6 +6,7 @@ TemporaryDirectory, partialmethod, to_native_string, + StringIO, ) from .contextmanagers import ( atomic_open_for_write, @@ -14,6 +15,7 @@ temp_environ, temp_path, spinner, + replaced_stream ) from .misc import ( load_path, @@ -26,6 +28,8 @@ take, chunked, divide, + get_wrapped_stream, + StreamWrapper ) from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile from .spin import VistirSpinner, create_spinner diff --git a/src/vistir/contextmanagers.py b/src/vistir/contextmanagers.py index 5e10e70..b2627d2 100644 --- a/src/vistir/contextmanagers.py +++ b/src/vistir/contextmanagers.py @@ -10,7 +10,7 @@ import six -from .compat import NamedTemporaryFile, Path, StringIO +from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path