Decode popen output using the system locale if UTF-8 decoding fails. (#…

…380)

* add decode_output func to decode popen output

fixes #309 by trying to decode popen output with utf8 first
and on error tries other encodings provided by the systems
preferences.

* simplified decode_output function

* fix linting issues

* fix double hard coded utf8

* change utf8 to utf-8

* add tests for decode_output

* fix linting issues

* simplify nested try...except block and fix nested exception msg

Co-authored-by: Claudio Jolowicz <cjolowicz@gmail.com>

* fix tests for simplified deode_output

Co-authored-by: Claudio Jolowicz <cjolowicz@gmail.com>

nox/popen.py

@@ -12,11 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import locale
 import subprocess
 import sys
 from typing import IO, Mapping, Sequence, Tuple, Union
 
 
+def decode_output(output: bytes) -> str:
+    """Try to decode the given bytes with encodings from the system.
+
+    :param output: output to decode
+    :raises UnicodeDecodeError: if all encodings fail
+    :return: decoded string
+    """
+    try:
+        return output.decode("utf-8")
+    except UnicodeDecodeError:
+        second_encoding = locale.getpreferredencoding()
+        if second_encoding.casefold() in ("utf8", "utf-8"):
+            raise
+
+        return output.decode(second_encoding)
+
+
 def popen(
     args: Sequence[str],
     env: Mapping[str, str] = None,
@@ -45,4 +63,4 @@ def popen(
 
     return_code = proc.wait()
 
-    return return_code, out.decode("utf-8") if out else ""
+    return return_code, decode_output(out) if out else ""

tests/test_command.py

@@ -18,6 +18,7 @@
 from unittest import mock
 
 import nox.command
+import nox.popen
 import pytest
 
 PYTHON = sys.executable
@@ -294,3 +295,43 @@ def test_custom_stderr_failed_command(capsys, tmpdir):
         tempfile_contents = stderr.read().decode("utf-8")
         assert "out" not in tempfile_contents
         assert "err" in tempfile_contents
+
+
+def test_output_decoding() -> None:
+    result = nox.popen.decode_output(b"abc")
+
+    assert result == "abc"
+
+
+def test_output_decoding_non_ascii() -> None:
+    result = nox.popen.decode_output("ü".encode("utf-8"))
+
+    assert result == "ü"
+
+
+def test_output_decoding_utf8_only_fail(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(nox.popen.locale, "getpreferredencoding", lambda: "utf8")
+
+    with pytest.raises(UnicodeDecodeError) as exc:
+        nox.popen.decode_output(b"\x95")
+
+    assert exc.value.encoding == "utf-8"
+
+
+def test_output_decoding_utf8_fail_cp1252_success(
+    monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(nox.popen.locale, "getpreferredencoding", lambda: "cp1252")
+
+    result = nox.popen.decode_output(b"\x95")
+
+    assert result == "•"  # U+2022
+
+
+def test_output_decoding_both_fail(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(nox.popen.locale, "getpreferredencoding", lambda: "ascii")
+
+    with pytest.raises(UnicodeDecodeError) as exc:
+        nox.popen.decode_output(b"\x95")
+
+    assert exc.value.encoding == "ascii"