Fix issue 326 unicode decode error #374

Merged
merged 5 commits into from Feb 7, 2012
View
13 pip/backwardcompat.py
@@ -61,7 +61,14 @@ def u(s):
return s.decode('utf-8')
def console_to_str(s):
- return s.decode(console_encoding)
+ try:
+ return s.decode(console_encoding)
+ except UnicodeDecodeError:
+ return s.decode('utf_8')
+
+ def fwrite(f, s):
+ f.buffer.write(b(s))
+
bytes = bytes
string_types = (str,)
raw_input = input
@@ -86,6 +93,10 @@ def u(s):
def console_to_str(s):
return s
+
+ def fwrite(f, s):
+ f.write(s)
+
bytes = str
string_types = (basestring,)
reduce = reduce
View
5 pip/log.py
@@ -4,6 +4,8 @@
import sys
import logging
+import pip.backwardcompat
+
class Logger(object):
@@ -71,7 +73,8 @@ def log(self, level, msg, *args, **kw):
## FIXME: should this be a name, not a level number?
rendered = '%02i %s' % (level, rendered)
if hasattr(consumer, 'write'):
- consumer.write(rendered+'\n')
+ rendered += '\n'
+ pip.backwardcompat.fwrite(consumer, rendered)
else:
consumer(rendered)
View
0 tests/packages/BrokenEmitsUTF8/broken.py
No changes.
View
25 tests/packages/BrokenEmitsUTF8/setup.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+from distutils.core import setup
+import sys
+
+class FakeError(Exception):
+ pass
+
+if sys.argv[1] == 'install':
+ if hasattr(sys.stdout, 'buffer'):
@carljm
carljm added a note Dec 12, 2011

Same here, let's encapsulate this into pip.backwardcompat.

@carljm
carljm added a note Dec 12, 2011

Er, never mind, given the context here (setup.py of a fake package), better leave it as is, not a problem.

OK, left this as-is.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ sys.stdout.buffer.write('\nThis package prints out UTF-8 stuff like:\n'.encode('utf-8'))
+ sys.stdout.buffer.write('* return type of ‘main’ is not ‘int’\n'.encode('utf-8'))
+ sys.stdout.buffer.write('* Björk Guðmundsdóttir [ˈpjœr̥k ˈkvʏðmʏntsˌtoʊhtɪr]'.encode('utf-8'))
+ else:
+ pass
+ sys.stdout.write('\nThis package prints out UTF-8 stuff like:\n')
+ sys.stdout.write('* return type of \xe2\x80\x98main\xe2\x80\x99 is not \xe2\x80\x98int\xe2\x80\x99\n')
+ sys.stdout.write('* Bj\xc3\xb6rk Gu\xc3\xb0mundsd\xc3\xb3ttir [\xcb\x88pj\xc5\x93r\xcc\xa5k \xcb\x88kv\xca\x8f\xc3\xb0m\xca\x8fnts\xcb\x8cto\xca\x8aht\xc9\xaar]\n')
+
+ raise FakeError('this package designed to fail on install')
+
+setup(name='broken',
+ version='0.2broken',
+ py_modules=['broken'],
+ )
View
25 tests/test_unicode.py
@@ -0,0 +1,25 @@
+import os
+from tests.test_pip import here, reset_env, run_pip
+
+
+def test_install_package_that_emits_unicode():
+ """
+ Install a package with a setup.py that emits UTF-8 output and then fails.
+ This works fine in Python 2, but fails in Python 3 with:
+
+ Traceback (most recent call last):
+ ...
+ File "/Users/marc/python/virtualenvs/py3.1-phpserialize/lib/python3.2/site-packages/pip-1.0.2-py3.2.egg/pip/__init__.py", line 230, in call_subprocess
+ line = console_to_str(stdout.readline())
+ File "/Users/marc/python/virtualenvs/py3.1-phpserialize/lib/python3.2/site-packages/pip-1.0.2-py3.2.egg/pip/backwardcompat.py", line 60, in console_to_str
+ return s.decode(console_encoding)
+ UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 17: ordinal not in range(128)
+
+ Refs https://github.com/pypa/pip/issues/326
+ """
+
+ env = reset_env()
+ to_install = os.path.abspath(os.path.join(here, 'packages', 'BrokenEmitsUTF8'))
+ result = run_pip('install', to_install, expect_error=True)
+ assert '__main__.FakeError: this package designed to fail on install' in result.stdout
+ assert 'UnicodeDecodeError' not in result.stdout