Browse files

Fixes for UltraTB and PyColorize with Python 3

  • Loading branch information...
1 parent d82fb26 commit f866f01c37d77d4027465a76f4eae07f0db9014b @takluyver committed Aug 7, 2011
Showing with 21 additions and 5 deletions.
  1. +10 −4 IPython/core/ultratb.py
  2. +9 −1 IPython/utils/PyColorize.py
  3. +2 −0 IPython/utils/py3compat.py
View
14 IPython/core/ultratb.py
@@ -83,6 +83,11 @@
import traceback
import types
+try: # Python 2
+ generate_tokens = tokenize.generate_tokens
+except AttributeError: # Python 3
+ generate_tokens = tokenize.tokenize
+
# For purposes of monkeypatching inspect to fix a bug in it.
from inspect import getsourcefile, getfile, getmodule,\
ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
@@ -94,6 +99,7 @@
from IPython.core.excolors import exception_colors
from IPython.utils import PyColorize
from IPython.utils import io
+from IPython.utils import py3compat
from IPython.utils.data import uniq_stable
from IPython.utils.warn import info, error
@@ -278,8 +284,7 @@ def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):
# serious refactoring, so that all of the ultratb and PyColorize code
# is unicode-safe. So for now this is rather an ugly hack, but
# necessary to at least have readable tracebacks. Improvements welcome!
- if type(line)==unicode:
- line = line.encode('utf-8', 'replace')
+ line = py3compat.cast_bytes_py2(line, 'utf-8')
new_line, err = _line_format(line, 'str', scheme)
if not err: line = new_line
@@ -872,7 +877,8 @@ def linereader(file=file, lnum=[lnum], getline=linecache.getline):
try:
# This builds the names list in-place by capturing it from the
# enclosing scope.
- tokenize.tokenize(linereader, tokeneater)
+ for token in generate_tokens(linereader):
+ tokeneater(*token)
except IndexError:
# signals exit of tokenizer
pass
@@ -933,7 +939,7 @@ def linereader(file=file, lnum=[lnum], getline=linecache.getline):
# ... and format it
exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
ColorsNormal, evalue_str)]
- if type(evalue) is types.InstanceType:
+ if (not py3compat.PY3) and type(evalue) is types.InstanceType:
try:
names = [w for w in dir(evalue) if isinstance(w, basestring)]
except:
View
10 IPython/utils/PyColorize.py
@@ -42,6 +42,13 @@
import token
import tokenize
+try:
+ generate_tokens = tokenize.generate_tokens
+except AttributeError:
+ # Python 3. Note that we use the undocumented _tokenize because it expects
+ # strings, not bytes. See also Python issue #9969.
+ generate_tokens = tokenize._tokenize
+
from IPython.utils.coloransi import *
#############################################################################
@@ -177,7 +184,8 @@ def format2(self, raw, out = None, scheme = ''):
error = False
try:
- tokenize.tokenize(text.readline, self)
+ for token in generate_tokens(text.readline):
+ self(*token)
except tokenize.TokenError, ex:
msg = ex[0]
line = ex[1][0]
View
2 IPython/utils/py3compat.py
@@ -35,6 +35,7 @@ def cast_bytes(s, encoding=None):
unicode_to_str = no_code
str_to_bytes = encode
bytes_to_str = decode
+ cast_bytes_py2 = no_code
def isidentifier(s, dotted=False):
if dotted:
@@ -53,6 +54,7 @@ def isidentifier(s, dotted=False):
unicode_to_str = encode
str_to_bytes = no_code
bytes_to_str = no_code
+ cast_bytes_py2 = cast_bytes
import re
_name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")

0 comments on commit f866f01

Please sign in to comment.