Fix issue 326 unicode decode error

pip/backwardcompat.py

@@ -61,7 +61,14 @@ def u(s):
         return s.decode('utf-8')
 
     def console_to_str(s):
-        return s.decode(console_encoding)
+        try:
+            return s.decode(console_encoding)
+        except UnicodeDecodeError:
+            return s.decode('utf_8')
+
+    def fwrite(f, s):
+        f.buffer.write(b(s))
+
     bytes = bytes
     string_types = (str,)
     raw_input = input
@@ -86,6 +93,10 @@ def u(s):
 
     def console_to_str(s):
         return s
+
+    def fwrite(f, s):
+        f.write(s)
+
     bytes = str
     string_types = (basestring,)
     reduce = reduce

pip/log.py

@@ -4,6 +4,8 @@
 import sys
 import logging
 
+import pip.backwardcompat
+
 
 class Logger(object):
 
@@ -71,7 +73,8 @@ def log(self, level, msg, *args, **kw):
                         ## FIXME: should this be a name, not a level number?
                         rendered = '%02i %s' % (level, rendered)
                 if hasattr(consumer, 'write'):
-                    consumer.write(rendered+'\n')
+                    rendered += '\n'
+                    pip.backwardcompat.fwrite(consumer, rendered)
                 else:
                     consumer(rendered)
 

tests/packages/BrokenEmitsUTF8/setup.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+from distutils.core import setup
+import sys
+
+class FakeError(Exception):
+    pass
+
+if sys.argv[1] == 'install':
+    if hasattr(sys.stdout, 'buffer'):
+        sys.stdout.buffer.write('\nThis package prints out UTF-8 stuff like:\n'.encode('utf-8'))
+        sys.stdout.buffer.write('* return type of ‘main’ is not ‘int’\n'.encode('utf-8'))
+        sys.stdout.buffer.write('* Björk Guðmundsdóttir [ˈpjœr̥k ˈkvʏðmʏntsˌtoʊhtɪr]'.encode('utf-8'))
+    else:
+        pass
+        sys.stdout.write('\nThis package prints out UTF-8 stuff like:\n')
+        sys.stdout.write('* return type of \xe2\x80\x98main\xe2\x80\x99 is not \xe2\x80\x98int\xe2\x80\x99\n')
+        sys.stdout.write('* Bj\xc3\xb6rk Gu\xc3\xb0mundsd\xc3\xb3ttir [\xcb\x88pj\xc5\x93r\xcc\xa5k \xcb\x88kv\xca\x8f\xc3\xb0m\xca\x8fnts\xcb\x8cto\xca\x8aht\xc9\xaar]\n')
+
+    raise FakeError('this package designed to fail on install')
+
+setup(name='broken',
+      version='0.2broken',
+      py_modules=['broken'],
+      )

tests/test_unicode.py

@@ -0,0 +1,25 @@
+import os
+from tests.test_pip import here, reset_env, run_pip
+
+
+def test_install_package_that_emits_unicode():
+    """
+    Install a package with a setup.py that emits UTF-8 output and then fails.
+    This works fine in Python 2, but fails in Python 3 with:
+
+    Traceback (most recent call last):
+      ...
+      File "/Users/marc/python/virtualenvs/py3.1-phpserialize/lib/python3.2/site-packages/pip-1.0.2-py3.2.egg/pip/__init__.py", line 230, in call_subprocess
+        line = console_to_str(stdout.readline())
+      File "/Users/marc/python/virtualenvs/py3.1-phpserialize/lib/python3.2/site-packages/pip-1.0.2-py3.2.egg/pip/backwardcompat.py", line 60, in console_to_str
+        return s.decode(console_encoding)
+    UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 17: ordinal not in range(128)
+
+    Refs https://github.com/pypa/pip/issues/326
+    """
+
+    env = reset_env()
+    to_install = os.path.abspath(os.path.join(here, 'packages', 'BrokenEmitsUTF8'))
+    result = run_pip('install', to_install, expect_error=True)
+    assert '__main__.FakeError: this package designed to fail on install' in result.stdout
+    assert 'UnicodeDecodeError' not in result.stdout