Overhaul exception reporting.

unittest2 recently added the ability to show local variables in
tracebacks as #111
requested for us. Reusing that requires some refactoring of our code,
in particular where we were reimplementing bits of the traceback
module. Now we can just hard-depend on traceback2 and linecache2 which
are brought in by unittest2 1.0.0.

Change-Id: Ieb3268029d26b48ed4fcd25ed644bd339f6aa3fb

NEWS

@@ -7,6 +7,15 @@ Changes and improvements to testtools_, grouped by release.
 NEXT
 ~~~~
 
+Improvements
+------------
+
+* ``testtools.run`` now accepts ``--locals`` to show local variables
+  in tracebacks, which can be a significant aid in debugging. In doing
+  so we've removed the code reimplementing linecache and traceback as
+  the traceback2 and linecache2 modules can now be used instead.
+  (Robert Collins, github #111)
+
 1.5.0
 ~~~~~
 

doc/for-framework-folk.rst

@@ -92,6 +92,13 @@ cause ``testtools.RunTest`` to fail the test case after the test has finished.
 This is useful when you want to cause a test to fail, but don't want to
 prevent the remainder of the test code from being executed.
 
+Exception formatting
+--------------------
+
+Testtools ``TestCase`` do their own formatting of exceptions. The attribute
+``__testtools_tb_locals__`` controls whether to include local variables in the
+formatted exceptions.
+
 Test placeholders
 =================
 

doc/for-test-authors.rst

@@ -72,10 +72,12 @@ installed or have Python 2.7 or later, and then run::
 
   $ python -m testtools.run discover packagecontainingtests
 
-For more information see the Python 2.7 unittest documentation, or::
+For more information see the Python unittest documentation, and::
 
     python -m testtools.run --help
 
+which describes the options available to ``testtools.run``.
+
 As your testing needs grow and evolve, you will probably want to use a more
 sophisticated test runner.  There are many of these for Python, and almost all
 of them will happily run testtools tests.  In particular:

setup.py

@@ -63,7 +63,8 @@ def get_long_description():
     # 'mimeparse' has not been uploaded by the maintainer with Python3 compat
     # but someone kindly uploaded a fixed version as 'python-mimeparse'.
     'python-mimeparse',
-    'unittest2>=0.8.0',
+    'unittest2>=1.0.0',
+    'traceback2',
     ]
 
 

testtools/compat.py

@@ -18,18 +18,19 @@
 
 import codecs
 import io
-import linecache
 import locale
 import os
 import re
 import sys
 import traceback
 import unicodedata
 
-from extras import try_imports
+from extras import try_import, try_imports
 
 BytesIO = try_imports(['StringIO.StringIO', 'io.BytesIO'])
 StringIO = try_imports(['StringIO.StringIO', 'io.StringIO'])
+# To let setup.py work, make this a conditional import.
+linecache = try_import('linecache2')
 
 try:
     from testtools import _compat2x as _compat
@@ -209,61 +210,6 @@ def unicode_output_stream(stream):
         except AttributeError:
             pass
     return writer(stream, "replace")
-
-
-# The default source encoding is actually "iso-8859-1" until Python 2.5 but
-# using non-ascii causes a deprecation warning in 2.4 and it's cleaner to
-# treat all versions the same way
-_default_source_encoding = "ascii"
-
-# Pattern specified in <http://www.python.org/dev/peps/pep-0263/>
-_cookie_search=re.compile("coding[:=]\s*([-\w.]+)").search
-
-def _detect_encoding(lines):
-    """Get the encoding of a Python source file from a list of lines as bytes
-
-    This function does less than tokenize.detect_encoding added in Python 3 as
-    it does not attempt to raise a SyntaxError when the interpreter would, it
-    just wants the encoding of a source file Python has already compiled and
-    determined is valid.
-    """
-    if not lines:
-        return _default_source_encoding
-    if lines[0].startswith("\xef\xbb\xbf"):
-        # Source starting with UTF-8 BOM is either UTF-8 or a SyntaxError
-        return "utf-8"
-    # Only the first two lines of the source file are examined
-    magic = _cookie_search("".join(lines[:2]))
-    if magic is None:
-        return _default_source_encoding
-    encoding = magic.group(1)
-    try:
-        codecs.lookup(encoding)
-    except LookupError:
-        # Some codecs raise something other than LookupError if they don't
-        # support the given error handler, but not the text ones that could
-        # actually be used for Python source code
-        return _default_source_encoding
-    return encoding
-
-
-class _EncodingTuple(tuple):
-    """A tuple type that can have an encoding attribute smuggled on"""
-
-
-def _get_source_encoding(filename):
-    """Detect, cache and return the encoding of Python source at filename"""
-    try:
-        return linecache.cache[filename].encoding
-    except (AttributeError, KeyError):
-        encoding = _detect_encoding(linecache.getlines(filename))
-        if filename in linecache.cache:
-            newtuple = _EncodingTuple(linecache.cache[filename])
-            newtuple.encoding = encoding
-            linecache.cache[filename] = newtuple
-        return encoding
-
-
 def _get_exception_encoding():
     """Return the encoding we expect messages from the OS to be encoded in"""
     if os.name == "nt":
@@ -276,110 +222,3 @@ def _get_exception_encoding():
     return locale.getlocale(locale.LC_MESSAGES)[1] or "ascii"
 
 
-def _exception_to_text(evalue):
-    """Try hard to get a sensible text value out of an exception instance"""
-    try:
-        return unicode(evalue)
-    except KeyboardInterrupt:
-        raise
-    except:
-        # Apparently this is what traceback._some_str does. Sigh - RBC 20100623
-        pass
-    try:
-        return str(evalue).decode(_get_exception_encoding(), "replace")
-    except KeyboardInterrupt:
-        raise
-    except:
-        # Apparently this is what traceback._some_str does. Sigh - RBC 20100623
-        pass
-    # Okay, out of ideas, let higher level handle it
-    return None
-
-
-def _format_stack_list(stack_lines):
-    """Format 'stack_lines' and return a list of unicode strings.
-
-    :param stack_lines: A list of filename, lineno, name, and line variables,
-        probably obtained by calling traceback.extract_tb or
-        traceback.extract_stack.
-    """
-    fs_enc = sys.getfilesystemencoding()
-    extracted_list = []
-    for filename, lineno, name, line in stack_lines:
-            extracted_list.append((
-                filename.decode(fs_enc, "replace"),
-                lineno,
-                name.decode("ascii", "replace"),
-                line and line.decode(
-                    _get_source_encoding(filename), "replace")))
-    return traceback.format_list(extracted_list)
-
-
-def _format_exception_only(eclass, evalue):
-    """Format the excption part of a traceback.
-
-    :param eclass: The type of the exception being formatted.
-    :param evalue: The exception instance.
-    :returns: A list of unicode strings.
-    """
-    list = []
-    if evalue is None:
-        # Is a (deprecated) string exception
-        list.append((eclass + "\n").decode("ascii", "replace"))
-        return list
-    if isinstance(evalue, SyntaxError):
-        # Avoid duplicating the special formatting for SyntaxError here,
-        # instead create a new instance with unicode filename and line
-        # Potentially gives duff spacing, but that's a pre-existing issue
-        try:
-            msg, (filename, lineno, offset, line) = evalue
-        except (TypeError, ValueError):
-            pass # Strange exception instance, fall through to generic code
-        else:
-            # Errors during parsing give the line from buffer encoded as
-            # latin-1 or utf-8 or the encoding of the file depending on the
-            # coding and whether the patch for issue #1031213 is applied, so
-            # give up on trying to decode it and just read the file again
-            if line:
-                bytestr = linecache.getline(filename, lineno)
-                if bytestr:
-                    if lineno == 1 and bytestr.startswith("\xef\xbb\xbf"):
-                        bytestr = bytestr[3:]
-                    line = bytestr.decode(
-                        _get_source_encoding(filename), "replace")
-                    del linecache.cache[filename]
-                else:
-                    line = line.decode("ascii", "replace")
-            if filename:
-                fs_enc = sys.getfilesystemencoding()
-                filename = filename.decode(fs_enc, "replace")
-            evalue = eclass(msg, (filename, lineno, offset, line))
-            list.extend(traceback.format_exception_only(eclass, evalue))
-            return list
-    sclass = eclass.__name__
-    svalue = _exception_to_text(evalue)
-    if svalue:
-        list.append("%s: %s\n" % (sclass, svalue))
-    elif svalue is None:
-        # GZ 2010-05-24: Not a great fallback message, but keep for the moment
-        list.append(_u("%s: <unprintable %s object>\n" % (sclass, sclass)))
-    else:
-        list.append(_u("%s\n" % sclass))
-    return list
-
-
-_TB_HEADER = _u('Traceback (most recent call last):\n')
-
-
-def _format_exc_info(eclass, evalue, tb, limit=None):
-    """Format a stack trace and the exception information as unicode
-
-    Compatibility function for Python 2 which ensures each component of a
-    traceback is correctly decoded according to its origins.
-
-    Based on traceback.format_exception and related functions.
-    """
-    return [_TB_HEADER] \
-        + _format_stack_list(traceback.extract_tb(tb, limit)) \
-        + _format_exception_only(eclass, evalue)
-