Fix getting unicode lines in IPython.core.debugger.

IPython/core/debugger.py

@@ -30,7 +30,7 @@
 import linecache
 import sys
 
-from IPython.utils import PyColorize, py3compat
+from IPython.utils import PyColorize, ulinecache
 from IPython.core import ipapi
 from IPython.utils import coloransi, io, openpy
 from IPython.core.excolors import exception_colors
@@ -179,22 +179,6 @@ def _file_lines(fname):
         return out
 
 
-def _readline(x):
-    """helper to pop elements off list of string
-
-    call with list of strings, return readline function that will pop
-    one line off the beginning of a copy of the list with each call.
-    raise StopIteration when empty or on third call
-    """
-    x = x[:2]
-    def readline():
-        if x:
-            return x.pop(0)
-        else:
-            raise StopIteration
-    return readline
-
-
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline."""
 
@@ -320,7 +304,7 @@ def print_stack_entry(self,frame_lineno,prompt_prefix='\n-> ',
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
-        import linecache, repr
+        import repr
 
         ret = []
 
@@ -367,11 +351,7 @@ def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
         ret.append('%s(%s)%s\n' % (link,lineno,call))
 
         start = lineno - 1 - context//2
-        lines = linecache.getlines(filename)
-        try:
-            encoding, _ = openpy.detect_encoding(_readline(lines))
-        except SyntaxError:
-            encoding = "ascii"
+        lines = ulinecache.getlines(filename)
         start = max(start, 0)
         start = min(start, len(lines) - context)
         lines = lines[start : start + context]
@@ -382,7 +362,7 @@ def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
                       and tpl_line_em \
                       or tpl_line
             ret.append(self.__format_line(linetpl, filename,
-                                          start + 1 + i, py3compat.cast_unicode(line),
+                                          start + 1 + i, line,
                                           arrow = show_arrow) )
         return ''.join(ret)
 
@@ -442,18 +422,10 @@ def print_list_lines(self, filename, first, last):
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             if filename == "<string>" and hasattr(self, "_exec_filename"):
-                lines = list(open(self._exec_filename))
-            else:
-                lines = linecache.getlines(filename)
-            try:
-                encoding, _ = openpy.detect_encoding(_readline(lines))
-            except SyntaxError:
-                encoding = "ascii"
-            if not lines:
-                print >>io.stdout, "No src could be located using filename: %r"%filename
-                return #Bailing out, there is nothing to see here
+                filename = self._exec_filename
+
             for lineno in range(first, last+1):
-                line = py3compat.cast_unicode(lines[lineno])
+                ulinecache.getline(filename, lineno)
                 if not line:
                     break
 

IPython/utils/openpy.py

@@ -208,3 +208,12 @@ def read_py_url(url, errors='replace', skip_encoding_cookie=True):
     response = urllib.urlopen(url)
     buffer = io.BytesIO(response.read())
     return source_to_unicode(buffer, errors, skip_encoding_cookie)
+
+def _list_readline(x):
+    """Given a list, returns a readline() function that returns the next element
+    with each call.
+    """
+    x = iter(x)
+    def readline():
+        return next(x)
+    return readline

IPython/utils/ulinecache.py

@@ -0,0 +1,38 @@
+"""Wrapper around linecache which decodes files to unicode according to PEP 263.
+
+This is only needed for Python 2 - linecache in Python 3 does the same thing
+itself.
+"""
+import functools
+import linecache
+
+from IPython.utils import py3compat
+from IPython.utils import openpy
+
+if py3compat.PY3:
+    getline = linecache.getline
+
+    # getlines has to be looked up at runtime, because doctests monkeypatch it.
+    @functools.wraps(linecache.getlines)
+    def getlines(filename, module_globals=None):
+        return linecache.getlines(filename, module_globals=module_globals)
+
+else:
+    def getlines(filename, module_globals=None):
+        """Get the lines (as unicode) for a file from the cache.
+        Update the cache if it doesn't contain an entry for this file already."""
+        linesb = linecache.getlines(filename, module_globals=module_globals)
+        readline = openpy._list_readline(linesb)
+        try:
+            encoding, _ = openpy.detect_encoding(readline)
+        except SyntaxError:
+            encoding = 'ascii'
+        return [l.decode(encoding, 'replace') for l in linesb]
+
+    # This is a straight copy of linecache.getline
+    def getline(filename, lineno, module_globals=None):
+        lines = getlines(filename, module_globals)
+        if 1 <= lineno <= len(lines):
+            return lines[lineno-1]
+        else:
+            return ''