diff --git a/.travis.yml b/.travis.yml index 9ea281a..4dd2f0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - "pipenv install --dev" - "pipenv run pip install --upgrade -e .[spinner,tests]" script: - - "pipenv run pytest -v -n auto tests/" + - "pipenv run pytest -v tests/" jobs: include: @@ -33,7 +33,7 @@ jobs: - stage: coverage python: "3.6" install: - - "pip install --upgrade pip pipenv pytest-cov pytest-xdist pytest-timeout pytest" + - "pip install --upgrade pip pipenv pytest-cov pytest-timeout pytest" - "pipenv install --dev" script: - - "pipenv run pytest -n auto --timeout 300 --cov=vistir --cov-report=term-missing --cov-report=xml --cov-report=html tests" + - "pipenv run pytest --timeout 300 --cov=vistir --cov-report=term-missing --cov-report=xml --cov-report=html tests" diff --git a/README.rst b/README.rst index 198f354..33cf53c 100644 --- a/README.rst +++ b/README.rst @@ -131,6 +131,7 @@ default encoding: * ``vistir.contextmanagers.atomic_open_for_write`` * ``vistir.contextmanagers.cd`` * ``vistir.contextmanagers.open_file`` + * ``vistir.contextmanagers.replaced_stream`` * ``vistir.contextmanagers.spinner`` * ``vistir.contextmanagers.temp_environ`` * ``vistir.contextmanagers.temp_path`` @@ -203,6 +204,23 @@ to pair this with an iterator which employs a sensible chunk size. shutil.copyfileobj(fp, filecontents) +.. _`replaced_stream`: + +A context manager to temporarily swap out *stream_name* with a stream wrapper. This will +capture the stream output and prevent it from being written as normal. + +.. code-block:: python + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + ... assert orig_stdout.getvalue() != "hello" + + >>> sys.stdout.write("hello") + 'hello' + + .. _`spinner`: **spinner** @@ -286,8 +304,13 @@ The following Miscellaneous utilities are available as helper methods: * ``vistir.misc.partialclass`` * ``vistir.misc.to_text`` * ``vistir.misc.to_bytes`` + * ``vistir.misc.divide`` + * ``vistir.misc.take`` + * ``vistir.misc.chunked`` * ``vistir.misc.decode_for_output`` - + * ``vistir.misc.get_canonical_encoding_name`` + * ``vistir.misc.get_wrapped_stream`` + * ``vistir.misc.StreamWrapper`` .. _`shell_escape`: @@ -401,6 +424,62 @@ Converts arbitrary byte-convertable input to bytes while handling errors. b'this is some text' +.. _`chunked`: + +**chunked** +//////////// + +Splits an iterable up into groups *of the specified length*, per `more itertools`_. Returns an iterable. + +This example will create groups of chunk size **5**, which means there will be *6 groups*. + +.. code-block:: python + + >>> chunked_iterable = vistir.misc.chunked(5, range(30)) + >>> for chunk in chunked_iterable: + ... add_to_some_queue(chunk) + +.. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + +.. _`take`: + +**take** +///////// + +Take elements from the supplied iterable without consuming it. + +.. code-block:: python + + >>> iterable = range(30) + >>> first_10 = take(10, iterable) + >>> [i for i in first_10] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> [i for i in iterable] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + + +.. _`divide`: + +**divide** +//////////// + +Splits an iterable up into the *specified number of groups*, per `more itertools`_. Returns an iterable. + +.. code-block:: python + + >>> iterable = range(30) + >>> groups = [] + >>> for grp in vistir.misc.divide(3, iterable): + ... groups.append(grp) + >>> groups + [, , ] + + +.. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + .. _`decode_for_output`: **decode_for_output** @@ -411,6 +490,46 @@ outputs using the system preferred locale using ``locale.getpreferredencoding(Fa with some additional hackery on linux systems. +.. _`get_canonical_encoding_name`: + +**get_canonical_encoding_name** +//////////////////////////////// + +Given an encoding name, get the canonical name from a codec lookup. + +.. code-block:: python + + >>> vistir.misc.get_canonical_encoding_name("utf8") + "utf-8" + + +.. _`get_wrapped_stream`: + +**get_wrapped_stream** +////////////////////// + +Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + +.. code-block:: python + + >>> stream = sys.stdout + >>> wrapped_stream = vistir.misc.get_wrapped_stream(sys.stdout) + + +.. _`StreamWrapper`: + +**StreamWrapper** +////////////////// + +A stream wrapper and compatibility class for handling wrapping file-like stream objects +which may be used in place of ``sys.stdout`` and other streams. + +.. code-block:: python + + >>> wrapped_stream = vistir.misc.StreamWrapper(sys.stdout, encoding="utf-8", errors="replace", line_buffering=True) + >>> wrapped_stream = vistir.misc.StreamWrapper(io.StringIO(), encoding="utf-8", errors="replace", line_buffering=True) + + 🐉 Path Utilities ------------------ diff --git a/appveyor.yml b/appveyor.yml index d07c3d8..1f377ae 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,9 +13,9 @@ environment: install: - "SET PATH=%PYTHON%\\;%PYTHON%\\Scripts;%PATH%" - "python --version" - - "python -m pip install --upgrade pip pipenv" + - "python -m pip install --upgrade pip pipenv pytest pytest-timeout pytest-cov" - "python -m pipenv install --dev" - - "python -m pipenv run pip install -e .[tests,spinner]" + - "python -m pipenv run pip install --upgrade -e .[tests,spinner]" build: off @@ -24,4 +24,4 @@ test_script: - "subst T: %TEMP%" - "set TEMP=T:\\" - "set TMP=T:\\" - - "python -m pipenv run pytest -n auto -v tests" + - "python -m pipenv run pytest -ra tests" diff --git a/docs/conf.py b/docs/conf.py index df13239..8e3007b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,11 +12,30 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import codecs import os +import re import sys docs_dir = os.path.abspath(os.path.dirname(__file__)) src_dir = os.path.join(os.path.dirname(docs_dir), "src", "vistir") sys.path.insert(0, src_dir) +version_file = os.path.join(src_dir, "__init__.py") + + +def read_file(path): + # intentionally *not* adding an encoding option to open, See: + # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 + with codecs.open(path, 'r') as fp: + return fp.read() + + +def find_version(file_path): + version_file = read_file(file_path) + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + return '0.0.0' # -- Project information ----------------------------------------------------- @@ -25,10 +44,12 @@ copyright = '2018, Dan Ryan ' author = 'Dan Ryan ' +release = find_version(version_file) +version = '.'.join(release.split('.')[:2]) # The short X.Y version -version = '0.0' +# version = '0.0' # The full version, including alpha/beta/rc tags -release = '0.0.0.dev0' +# release = '0.0.0.dev0' # -- General configuration --------------------------------------------------- @@ -132,11 +153,11 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # - # 'papersize': 'letterpaper', + 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # - # 'pointsize': '10pt', + 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # @@ -173,7 +194,7 @@ # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'vistir', 'vistir Documentation', - author, 'vistir', 'One line description of project.', + author, 'vistir', 'Miscellaneous utilities for dealing with filesystems, paths, projects, subprocesses, and more.', 'Miscellaneous'), ] diff --git a/docs/quickstart.rst b/docs/quickstart.rst index b9d0102..8560a7d 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -131,6 +131,7 @@ defualt encoding: * :func:`~vistir.contextmanagers.atomic_open_for_write` * :func:`~vistir.contextmanagers.cd` * :func:`~vistir.contextmanagers.open_file` + * :func:`~vistir.contextmanagers.replaced_stream` * :func:`~vistir.contextmanagers.spinner` * :func:`~vistir.contextmanagers.temp_environ` * :func:`~vistir.contextmanagers.temp_path` @@ -203,6 +204,22 @@ to pair this with an iterator which employs a sensible chunk size. shutil.copyfileobj(fp, filecontents) +.. _`replaced_stream`: + +A context manager to temporarily swap out *stream_name* with a stream wrapper. This will +capture the stream output and prevent it from being written as normal. + +.. code-block:: python + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + + >>> sys.stdout.write("hello") + 'hello' + + .. _`spinner`: **spinner** @@ -286,7 +303,13 @@ The following Miscellaneous utilities are available as helper methods: * :func:`~vistir.misc.partialclass` * :func:`~vistir.misc.to_text` * :func:`~vistir.misc.to_bytes` + * :func:`~vistir.misc.divide` + * :func:`~vistir.misc.take` + * :func:`~vistir.misc.chunked` * :func:`~vistir.misc.decode_for_output` + * :func:`~vistir.misc.get_canonical_encoding_name` + * :func:`~vistir.misc.get_wrapped_stream` + * :class:`~vistir.misc.StreamWrapper` .. _`shell_escape`: @@ -401,6 +424,62 @@ Converts arbitrary byte-convertable input to bytes while handling errors. b'this is some text' +.. _`chunked`: + +**chunked** +//////////// + +Splits an iterable up into groups *of the specified length*, per `more itertools`_. Returns an iterable. + +This example will create groups of chunk size **5**, which means there will be *6 groups*. + +.. code-block:: python + + >>> chunked_iterable = vistir.misc.chunked(5, range(30)) + >>> for chunk in chunked_iterable: + ... add_to_some_queue(chunk) + +.. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + +.. _`take`: + +**take** +///////// + +Take elements from the supplied iterable without consuming it. + +.. code-block:: python + + >>> iterable = range(30) + >>> first_10 = take(10, iterable) + >>> [i for i in first_10] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> [i for i in iterable] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + + +.. _`divide`: + +**divide** +//////////// + +Splits an iterable up into the *specified number of groups*, per `more itertools`_. Returns an iterable. + +.. code-block:: python + + >>> iterable = range(30) + >>> groups = [] + >>> for grp in vistir.misc.divide(3, iterable): + ... groups.append(grp) + >>> groups + [, , ] + + +.. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + .. _`decode_for_output`: **decode_for_output** @@ -416,6 +495,46 @@ with some additional hackery on linux systems. "some default locale encoded text" +.. _`get_canonical_encoding_name`: + +**get_canonical_encoding_name** +//////////////////////////////// + +Given an encoding name, get the canonical name from a codec lookup. + +.. code-block:: python + + >>> vistir.misc.get_canonical_encoding_name("utf8") + "utf-8" + + +.. _`get_wrapped_stream`: + +**get_wrapped_stream** +////////////////////// + +Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + +.. code-block:: python + + >>> stream = sys.stdout + >>> wrapped_stream = vistir.misc.get_wrapped_stream(sys.stdout) + + +.. _`StreamWrapper`: + +**StreamWrapper** +////////////////// + +A stream wrapper and compatibility class for handling wrapping file-like stream objects +which may be used in place of ``sys.stdout`` and other streams. + +.. code-block:: python + + >>> wrapped_stream = vistir.misc.StreamWrapper(sys.stdout, encoding="utf-8", errors="replace", line_buffering=True) + >>> wrapped_stream = vistir.misc.StreamWrapper(io.StringIO(), encoding="utf-8", errors="replace", line_buffering=True) + + 🐉 Path Utilities ------------------ diff --git a/news/48.feature.rst b/news/48.feature.rst new file mode 100644 index 0000000..92cf00e --- /dev/null +++ b/news/48.feature.rst @@ -0,0 +1,3 @@ +Added a new ``vistir.misc.StreamWrapper`` class with ``vistir.misc.get_wrapped_stream()`` to wrap existing streams +and ``vistir.contextmanagers.replaced_stream()`` to temporarily replace a stream. + diff --git a/news/49.bugfix.rst b/news/49.bugfix.rst new file mode 100644 index 0000000..a8c2be7 --- /dev/null +++ b/news/49.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug with exception handling during ``_create_process`` calls. diff --git a/news/51.feature.rst b/news/51.feature.rst new file mode 100644 index 0000000..b51bfea --- /dev/null +++ b/news/51.feature.rst @@ -0,0 +1 @@ +Added new entries in ``vistir.compat`` to support movements to ``collections.abc``: ``Mapping``, ``Sequence``, ``Set``, ``ItemsView``. diff --git a/news/52.feature.rst b/news/52.feature.rst new file mode 100644 index 0000000..ff2b556 --- /dev/null +++ b/news/52.feature.rst @@ -0,0 +1,2 @@ +Improved ``decode_for_output`` to handle decoding failures gracefully by moving to an ``replace`` strategy. +Now also allows a translation map to be provided to translate specific non-ascii characters when writing to outputs. diff --git a/setup.cfg b/setup.cfg index 55026d5..9626be1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,18 +31,22 @@ classifier = [options] zip_safe = true python_requires = >=2.6,!=3.0,!=3.1,!=3.2,!=3.3 -setup_requires = setuptools>=36.2.2 +setup_requires = + setuptools>=38.2.5 + invoke + parver + wheel install_requires = - pathlib2;python_version<"3.5" - backports.functools_lru_cache;python_version <= "3.4" + colorama + backports.functools_lru_cache;python_version<="3.4" backports.shutil_get_terminal_size;python_version<"3.3" backports.weakref;python_version<"3.3" + pathlib2;python_version<"3.5" requests six [options.extras_require] spinner = - colorama cursor yaspin tests = diff --git a/src/vistir/__init__.py b/src/vistir/__init__.py index f54a309..b7fc190 100644 --- a/src/vistir/__init__.py +++ b/src/vistir/__init__.py @@ -6,6 +6,7 @@ TemporaryDirectory, partialmethod, to_native_string, + StringIO, ) from .contextmanagers import ( atomic_open_for_write, @@ -14,6 +15,7 @@ temp_environ, temp_path, spinner, + replaced_stream ) from .misc import ( load_path, @@ -26,6 +28,8 @@ take, chunked, divide, + get_wrapped_stream, + StreamWrapper ) from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile from .spin import VistirSpinner, create_spinner @@ -61,4 +65,8 @@ "take", "chunked", "divide", + "StringIO", + "get_wrapped_stream", + "StreamWrapper", + "replaced_stream" ] diff --git a/src/vistir/compat.py b/src/vistir/compat.py index 0f91855..8f957ec 100644 --- a/src/vistir/compat.py +++ b/src/vistir/compat.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import errno import os @@ -26,6 +26,11 @@ "TemporaryDirectory", "NamedTemporaryFile", "to_native_string", + "Iterable", + "Mapping", + "Sequence", + "Set", + "ItemsView" ] if sys.version_info >= (3, 5): @@ -46,20 +51,22 @@ try: from weakref import finalize except ImportError: - from backports.weakref import finalize + from backports.weakref import finalize # type: ignore try: from functools import partialmethod except Exception: - from .backports.functools import partialmethod + from .backports.functools import partialmethod # type: ignore try: from json import JSONDecodeError except ImportError: # Old Pythons. - JSONDecodeError = ValueError + JSONDecodeError = ValueError # type: ignore if six.PY2: + from io import BytesIO as StringIO + class ResourceWarning(Warning): pass @@ -79,13 +86,24 @@ class IsADirectoryError(OSError): """The command does not work on directories""" pass -else: - from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError - - -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable + class FileExistsError(OSError): + def __init__(self, *args, **kwargs): + self.errno = errno.EEXIST + super(FileExistsError, self).__init__(*args, **kwargs) +else: + from builtins import ( + ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError, + FileExistsError + ) + from io import StringIO + +six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) # type: ignore +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) # type: ignore +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # type: ignore +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore +six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) # type: ignore +from six.moves import Iterable, Mapping, Sequence, Set, ItemsView # type: ignore # noqa if not sys.warnoptions: warnings.simplefilter("default", ResourceWarning) diff --git a/src/vistir/contextmanagers.py b/src/vistir/contextmanagers.py index 77fbb9d..b2627d2 100644 --- a/src/vistir/contextmanagers.py +++ b/src/vistir/contextmanagers.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function import io import os @@ -15,7 +15,8 @@ __all__ = [ - "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" + "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner", + "dummy_spinner", "replaced_stream" ] @@ -286,3 +287,56 @@ def open_file(link, session=None, stream=True): if conn is not None: conn.close() result.close() + + +@contextmanager +def replaced_stream(stream_name): + """ + Context manager to temporarily swap out *stream_name* with a stream wrapper. + + :param str stream_name: The name of a sys stream to wrap + :returns: A ``StreamWrapper`` replacement, temporarily + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + + >>> sys.stdout.write("hello") + 'hello' + """ + orig_stream = getattr(sys, stream_name) + new_stream = six.StringIO() + try: + setattr(sys, stream_name, new_stream) + yield getattr(sys, stream_name) + finally: + setattr(sys, stream_name, orig_stream) + + +@contextmanager +def replaced_streams(): + """ + Context manager to replace both ``sys.stdout`` and ``sys.stderr`` using + ``replaced_stream`` + + returns: *(stdout, stderr)* + + >>> import sys + >>> with vistir.contextmanagers.replaced_streams() as streams: + >>> stdout, stderr = streams + >>> sys.stderr.write("test") + >>> sys.stdout.write("hello") + >>> assert stdout.getvalue() == "hello" + >>> assert stderr.getvalue() == "test" + + >>> stdout.getvalue() + 'hello' + + >>> stderr.getvalue() + 'test' + """ + + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + yield (stdout, stderr) diff --git a/src/vistir/misc.py b/src/vistir/misc.py index 8537be7..eb91d79 100644 --- a/src/vistir/misc.py +++ b/src/vistir/misc.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function +import io import json import logging import locale @@ -15,7 +16,7 @@ import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod, to_native_string, Iterable +from .compat import Path, fs_str, partialmethod, to_native_string, Iterable, StringIO from .contextmanagers import spinner as spinner if os.name != "nt": @@ -38,6 +39,9 @@ class WindowsError(OSError): "divide", "getpreferredencoding", "decode_for_output", + "get_canonical_encoding_name", + "get_wrapped_stream", + "StreamWrapper", ] @@ -159,7 +163,10 @@ def _create_subprocess( c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr) except Exception as exc: - sys.stderr.write("Error %s while executing command %s", exc, " ".join(cmd._parts)) + import traceback + formatted_tb = "".join(traceback.format_exception(*sys.exc_info())) + sys.stderr.write("Error while executing command %s:" % " ".join(cmd._parts)) + sys.stderr.write(formatted_tb) raise if not block: c.stdin.close() @@ -514,19 +521,184 @@ def getpreferredencoding(): PREFERRED_ENCODING = getpreferredencoding() -def decode_for_output(output): +def get_output_encoding(source_encoding): + """ + Given a source encoding, determine the preferred output encoding. + + :param str source_encoding: The encoding of the source material. + :returns: The output encoding to decode to. + :rtype: str + """ + + if source_encoding is not None: + if get_canonical_encoding_name(source_encoding) == 'ascii': + return 'utf-8' + return get_canonical_encoding_name(source_encoding) + return get_canonical_encoding_name(PREFERRED_ENCODING) + + +def _encode(output, encoding=None, errors=None, translation_map=None): + if encoding is None: + encoding = PREFERRED_ENCODING + try: + output = output.encode(encoding) + except (UnicodeDecodeError, UnicodeEncodeError): + if translation_map is not None: + if six.PY2: + output = unicode.translate( + to_text(output, encoding=encoding, errors=errors), translation_map + ) + else: + output = output.translate(translation_map) + else: + output = to_text(output, encoding=encoding, errors=errors) + except AttributeError: + pass + return output + + +def decode_for_output(output, target_stream=None, translation_map=None): """Given a string, decode it for output to a terminal :param str output: A string to print to a terminal + :param target_stream: A stream to write to, we will encode to target this stream if possible. + :param dict translation_map: A mapping of unicode character ordinals to replacement strings. :return: A re-encoded string using the preferred encoding :rtype: str """ if not isinstance(output, six.string_types): return output + encoding = None + if target_stream is not None: + encoding = getattr(target_stream, "encoding", None) + encoding = get_output_encoding(encoding) try: - output = output.encode(PREFERRED_ENCODING) - except AttributeError: - pass - output = output.decode(PREFERRED_ENCODING) - return output + output = _encode(output, encoding=encoding, translation_map=translation_map) + except (UnicodeDecodeError, UnicodeEncodeError): + output = _encode(output, encoding=encoding, errors="replace", + translation_map=translation_map) + return to_text(output, encoding=encoding, errors="replace") + + +def get_canonical_encoding_name(name): + # type: (str) -> str + """ + Given an encoding name, get the canonical name from a codec lookup. + + :param str name: The name of the codec to lookup + :return: The canonical version of the codec name + :rtype: str + """ + + import codecs + try: + codec = codecs.lookup(name) + except LookupError: + return name + else: + return codec.name + + +def get_wrapped_stream(stream): + """ + Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + + :param stream: A stream instance to wrap + :returns: A new, wrapped stream + :rtype: :class:`StreamWrapper` + """ + + if stream is None: + raise TypeError("must provide a stream to wrap") + encoding = getattr(stream, "encoding", None) + encoding = get_output_encoding(encoding) + return StreamWrapper(stream, encoding, "replace", line_buffering=True) + + +class StreamWrapper(io.TextIOWrapper): + + """ + This wrapper class will wrap a provided stream and supply an interface + for compatibility. + """ + + def __init__(self, stream, encoding, errors, line_buffering=True, **kwargs): + self._stream = stream = _StreamProvider(stream) + io.TextIOWrapper.__init__( + self, stream, encoding, errors, line_buffering=line_buffering, **kwargs + ) + + # borrowed from click's implementation of stream wrappers, see + # https://github.com/pallets/click/blob/6cafd32/click/_compat.py#L64 + if six.PY2: + def write(self, x): + if isinstance(x, (str, buffer, bytearray)): + try: + self.flush() + except Exception: + pass + return self.buffer.write(str(x)) + return io.TextIOWrapper.write(self, x) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __del__(self): + try: + self.detach() + except Exception: + pass + + def isatty(self): + return self._stream.isatty() + + +# More things borrowed from click, this is because we are using `TextIOWrapper` instead of +# just a normal StringIO +class _StreamProvider(object): + def __init__(self, stream): + self._stream = stream + super(_StreamProvider, self).__init__() + + def __getattr__(self, name): + return getattr(self._stream, name) + + def read1(self, size): + fn = getattr(self._stream, "read1", None) + if fn is not None: + return fn(size) + if six.PY2: + return self._stream.readline(size) + return self._stream.read(size) + + def readable(self): + fn = getattr(self._stream, "readable", None) + if fn is not None: + return fn() + try: + self._stream.read(0) + except Exception: + return False + return True + + def writable(self): + fn = getattr(self._stream, "writable", None) + if fn is not None: + return fn() + try: + self._stream.write(b"") + except Exception: + return False + return True + + def seekable(self): + fn = getattr(self._stream, "seekable", None) + if fn is not None: + return fn() + try: + self._stream.seek(self._stream.tell()) + except Exception: + return False + return True diff --git a/src/vistir/path.py b/src/vistir/path.py index 91a9d66..6e9a7f6 100644 --- a/src/vistir/path.py +++ b/src/vistir/path.py @@ -31,6 +31,7 @@ "get_converted_relative_path", "handle_remove_readonly", "normalize_path", + "is_in_path", "is_file_url", "is_readonly_path", "is_valid_url", diff --git a/src/vistir/spin.py b/src/vistir/spin.py index 7bdc85e..a927505 100644 --- a/src/vistir/spin.py +++ b/src/vistir/spin.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import functools import os @@ -8,41 +9,72 @@ import time import colorama -import cursor import six from .compat import to_native_string from .termcolors import COLOR_MAP, COLORS, colored, DISABLE_COLORS +from .misc import decode_for_output from io import StringIO try: import yaspin + import cursor except ImportError: yaspin = None Spinners = None + SpinBase = None + cursor = None else: - from yaspin.spinners import Spinners + import yaspin.spinners + import yaspin.core + Spinners = yaspin.spinners.Spinners + SpinBase = yaspin.core.Yaspin + +if os.name == "nt": + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.fail() + spinner.stop() + sys.exit(0) + +else: + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. -handler = None -if yaspin and os.name == "nt": - handler = yaspin.signal_handlers.default_handler -elif yaspin and os.name != "nt": - handler = yaspin.signal_handlers.fancy_handler + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.red.fail("✘") + spinner.stop() + sys.exit(0) CLEAR_LINE = chr(27) + "[K" +TRANSLATION_MAP = { + 10004: u"OK", + 10008: u"x", +} + + +decode_output = functools.partial(decode_for_output, translation_map=TRANSLATION_MAP) + class DummySpinner(object): def __init__(self, text="", **kwargs): - super(DummySpinner, self).__init__() if DISABLE_COLORS: colorama.init() - from .misc import decode_for_output - self.text = to_native_string(decode_for_output(text)) if text else "" + self.text = to_native_string(decode_output(text)) if text else "" self.stdout = kwargs.get("stdout", sys.stdout) self.stderr = kwargs.get("stderr", sys.stderr) self.out_buff = StringIO() self.write_to_stdout = kwargs.get("write_to_stdout", False) + super(DummySpinner, self).__init__() def __enter__(self): if self.text and self.text != "None": @@ -50,11 +82,11 @@ def __enter__(self): self.write(self.text) return self - def __exit__(self, exc_type, exc_val, traceback): + def __exit__(self, exc_type, exc_val, tb): if exc_type: import traceback - from .misc import decode_for_output - self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) + formatted_tb = traceback.format_exception(exc_type, exc_val, tb) + self.write_err("".join(formatted_tb)) self._close_output_buffer() return False @@ -76,56 +108,63 @@ def _close_output_buffer(self): pass def fail(self, exitcode=1, text="FAIL"): - from .misc import decode_for_output - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.write(decode_for_output(text)) + self.write(text) else: - self.write_err(decode_for_output(text)) + self.write_err(text) self._close_output_buffer() def ok(self, text="OK"): - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.stdout.write(self.text) + self.write(text) else: - self.stderr.write(self.text) + self.write_err(text) self._close_output_buffer() return 0 def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r")) + target.write(decode_output("\r", target_stream=target)) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output("{0}\n".format(text), target_stream=target)) target.write(CLEAR_LINE) self._show_cursor(target=target) def write(self, text=None): if not self.write_to_stdout: return self.write_err(text) - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stdout.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stdout.write(line) - self.stdout.write(CLEAR_LINE) + if not self.stdout.closed: + stdout = self.stdout + else: + stdout = sys.stdout + text = decode_output(text, target_stream=stdout) + stdout.write(decode_output("\r", target_stream=stdout)) + line = decode_output("{0}\n".format(text), target_stream=stdout) + stdout.write(line) + stdout.write(CLEAR_LINE) def write_err(self, text=None): - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stderr.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stderr.write(line) - self.stderr.write(CLEAR_LINE) + if not self.stderr.closed: + stderr = self.stderr + else: + if sys.stderr.closed: + print(text) + return + stderr = sys.stderr + text = decode_output(text, target_stream=stderr) + stderr.write(decode_output("\r", target_stream=stderr)) + line = decode_output("{0}\n".format(text), target_stream=stderr) + stderr.write(line) + stderr.write(CLEAR_LINE) @staticmethod def _hide_cursor(target=None): @@ -136,10 +175,11 @@ def _show_cursor(target=None): pass -base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner +if SpinBase is None: + SpinBase = DummySpinner -class VistirSpinner(base_obj): +class VistirSpinner(SpinBase): "A spinner class for handling spinners on windows and posix." def __init__(self, *args, **kwargs): @@ -181,9 +221,9 @@ def __init__(self, *args, **kwargs): self.out_buff = StringIO() self.write_to_stdout = write_to_stdout self.is_dummy = bool(yaspin is None) - if os.environ.get("ANSI_COLORS_DISABLED", False): - colorama.deinit() super(VistirSpinner, self).__init__(*args, **kwargs) + if DISABLE_COLORS: + colorama.deinit() def ok(self, text="OK", err=False): """Set Ok (success) finalizer to a spinner.""" @@ -206,38 +246,40 @@ def fail(self, text="FAIL", err=False): def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r")) + target.write(decode_output("\r")) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output("{0}\n".format(text))) target.write(CLEAR_LINE) self._show_cursor(target=target) def write(self, text): if not self.write_to_stdout: return self.write_err(text) - from .misc import to_text - sys.stdout.write("\r") - self.stdout.write(CLEAR_LINE) + stdout = self.stdout + if self.stdout.closed: + stdout = sys.stdout + stdout.write(decode_output("\r", target_stream=stdout)) + stdout.write(decode_output(CLEAR_LINE, target_stream=stdout)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) - self.stdout.write(text) - self.out_buff.write(to_text(text)) + text = decode_output("{0}\n".format(text), target_stream=stdout) + stdout.write(text) + self.out_buff.write(decode_output(text, target_stream=self.out_buff)) def write_err(self, text): """Write error text in the terminal without breaking the spinner.""" - from .misc import to_text - - self.stderr.write("\r") - self.stderr.write(CLEAR_LINE) + stderr = self.stderr + if self.stderr.closed: + stderr = sys.stderr + stderr.write(decode_output("\r", target_stream=stderr)) + stderr.write(decode_output(CLEAR_LINE, target_stream=stderr)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) + text = decode_output("{0}\n".format(text), target_stream=stderr) self.stderr.write(text) - self.out_buff.write(to_text(text)) + self.out_buff.write(decode_output(text, target_stream=self.out_buff)) def start(self): if self._sigmap: @@ -272,26 +314,22 @@ def stop(self): if target.isatty(): self._show_cursor(target=target) - if self.stderr and self.stderr != sys.stderr: - self.stderr.close() - if self.stdout and self.stdout != sys.stdout: - self.stdout.close() self.out_buff.close() def _freeze(self, final_text, err=False): """Stop spinner, compose last frame and 'freeze' it.""" if not final_text: final_text = "" - text = to_native_string(final_text) + target = self.stderr if err else self.stdout + if target.closed: + target = sys.stderr if err else sys.stdout + text = decode_output(final_text, target_stream=target) self._last_frame = self._compose_out(text, mode="last") # Should be stopped here, otherwise prints after # self._freeze call will mess up the spinner self.stop() - if err or not self.write_to_stdout: - self.stderr.write(self._last_frame) - else: - self.stdout.write(self._last_frame) + target.write(self._last_frame) def _compose_color_func(self): fn = functools.partial( @@ -305,19 +343,19 @@ def _compose_color_func(self): def _compose_out(self, frame, mode=None): # Ensure Unicode input - frame = to_native_string(frame) + frame = decode_output(frame) if self._text is None: self._text = "" - text = to_native_string(self._text) + text = decode_output(self._text) if self._color_func is not None: frame = self._color_func(frame) if self._side == "right": frame, text = text, frame # Mode if not mode: - out = to_native_string("\r{0} {1}".format(frame, text)) + out = decode_output("\r{0} {1}".format(frame, text)) else: - out = to_native_string("{0} {1}\n".format(frame, text)) + out = decode_output("{0} {1}\n".format(frame, text)) return out def _spin(self): diff --git a/tests/conftest.py b/tests/conftest.py index ec04db6..1556d2c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,2 +1,10 @@ # -*- coding=utf-8 -*- import pytest +from vistir.contextmanagers import replaced_stream + + +@pytest.fixture(scope="function") +def capture_streams(): + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + yield (stdout, stderr) diff --git a/tests/test_contextmanagers.py b/tests/test_contextmanagers.py index 25878d7..cc2a4d0 100644 --- a/tests/test_contextmanagers.py +++ b/tests/test_contextmanagers.py @@ -85,3 +85,11 @@ def test_open_file(tmpdir): for chunk in iter(lambda: fp.read(16384), b""): local_contents += chunk assert local_contents == filecontents.read() + + +def test_replace_stream(capsys): + with vistir.contextmanagers.replaced_stream("stdout") as stdout: + sys.stdout.write("hello") + assert stdout.getvalue() == "hello" + out, err = capsys.readouterr() + assert out.strip() != "hello" diff --git a/tests/test_path.py b/tests/test_path.py index 6b713cb..cac7769 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -32,7 +32,7 @@ def test_safe_expandvars(): @given(legal_path_chars(), legal_path_chars()) -@settings(suppress_health_check=(HealthCheck.filter_too_much,)) +@settings(suppress_health_check=(HealthCheck.filter_too_much,), deadline=500) def test_mkdir_p(base_dir, subdir): assume(not any((dir_name in ["", ".", "./", ".."] for dir_name in [base_dir, subdir]))) assume(not (os.path.relpath(subdir, start=base_dir) == ".")) diff --git a/tests/test_spinner.py b/tests/test_spinner.py index 0a15cd5..531163a 100644 --- a/tests/test_spinner.py +++ b/tests/test_spinner.py @@ -1,19 +1,30 @@ # -*- coding=utf-8 -*- -import vistir.spin +import pytest +from vistir.contextmanagers import replaced_stream import time -def test_spinner(capsys): - with vistir.spin.create_spinner(spinner_name="bouncingBar", text="Running...", nospin=False) as spinner: - time.sleep(3) - spinner.ok("Ok!") - out, err = capsys.readouterr() - assert out.strip().endswith("Ok!") - assert err.strip() == "" - with vistir.spin.create_spinner(spinner_name="bouncingBar", text="Running...", nospin=False, write_to_stdout=False) as spinner: - time.sleep(3) - spinner.ok("Ok!") - out, err = capsys.readouterr() - assert err.strip().endswith("Ok!") - assert out.strip() == "" +@pytest.mark.parametrize("nospin, write_to_stdout", ( + (True, True), (True, False), + (False, True), (False, False) +)) +def test_spinner(monkeypatch, nospin, write_to_stdout): + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + with monkeypatch.context() as m: + # m.setenv("VISTIR_DISABLE_COLORS", "1") + import vistir.spin + with vistir.spin.create_spinner( + spinner_name="bouncingBar", text="Running...", nospin=nospin, + write_to_stdout=write_to_stdout + ) as spinner: + time.sleep(3) + spinner.ok("Ok!") + out = stdout.getvalue().strip(vistir.spin.CLEAR_LINE).strip() + err = stderr.getvalue().strip(vistir.spin.CLEAR_LINE).strip() + if write_to_stdout: + assert "Ok!" in out.strip().splitlines()[-1], out + assert err.strip() == "", err + else: + assert "Ok!" in err.strip().splitlines()[-1], err