From 3c6e215c3fe33b7418738660efcb22f3984dda30 Mon Sep 17 00:00:00 2001 From: Christian Aichinger Date: Sun, 3 May 2015 12:48:13 +0200 Subject: [PATCH 1/2] Prevent UnicodeErrors when forwarding child stdout In invoke.runners.Local, the stdout/stderr of child processes is read and forwarded to the console (i.e. invoke's stdout/stderr). The incoming data is read with locale.getdefaultencoding(), while output happens with sys.stdout.encoding. If these two encodings differ, it may not be possible to write the received data to stdout, e.g. if an UTF-8 Yen character is received, but sys.stdout.encoding is 'latin1' or 'ascii'. If that happens, the resulting exception causes the thread to end, and the child process hangs due to the child filling its output buffer. This is mostly a concern on Windows, where the preferred encoding and the console I/O encoding often differ. Cf. https://github.com/pyinvoke/invoke/issues/241 --- invoke/runners.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/invoke/runners.py b/invoke/runners.py index e77463c5c..ed6836bfb 100644 --- a/invoke/runners.py +++ b/invoke/runners.py @@ -239,7 +239,10 @@ def get(): yield data for data in codecs.iterdecode(get(), encoding, errors='replace'): if not hide: - dst.write(data) + # Make sure no UnicodeError happens, even if the data is + # garbled (e.g. due to encoding mismatch with the child). + encoded_data = data.encode(dst.encoding, errors='replace') + dst.buffer.write(encoded_data) dst.flush() cap.append(data) From 741f3c9eeb7db52e8fcb79e80ba01c5f141a360a Mon Sep 17 00:00:00 2001 From: Christian Aichinger Date: Sun, 3 May 2015 20:59:05 +0200 Subject: [PATCH 2/2] Don't rely on sys.stdout.buffer being available Avoid accessing sys.stdout.buffer to write raw bytes. Manually decode the bytes and write them as string instead. This approach avoids problems during testing, when sys.stdout can be mocked. --- invoke/runners.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/invoke/runners.py b/invoke/runners.py index ed6836bfb..389767474 100644 --- a/invoke/runners.py +++ b/invoke/runners.py @@ -242,7 +242,8 @@ def get(): # Make sure no UnicodeError happens, even if the data is # garbled (e.g. due to encoding mismatch with the child). encoded_data = data.encode(dst.encoding, errors='replace') - dst.buffer.write(encoded_data) + clean_data = encoded_data.decode(dst.encoding) + dst.write(clean_data) dst.flush() cap.append(data)