Merged in support-python2.3 branch.

------------------------------------------------------------
Use --include-merged or -n0 to see merged revisions.

.bzrignore

@@ -1,3 +1,5 @@
 pythoscope.egg-info
 lib2to3/Grammar*.pickle
 lib2to3/PatternGrammar*.pickle
+build/
+dist/

lib2to3/pgen2/parse.py

@@ -34,8 +34,11 @@ def __reduce__(self):
         """Implemented so pickle can serialize this object.
 
         >>> import pickle
-        >>> pickle.loads(pickle.dumps(ParseError(1, 2, 3, 4)))
-        ParseError('1: type=2, value=3, context=4',)
+        >>> pe = pickle.loads(pickle.dumps(ParseError(1, 2, 3, 4)))
+        >>> isinstance(pe, ParseError)
+        True
+        >>> (pe.msg, pe.type, pe.value, pe.context) == (1, 2, 3, 4)
+        True
         """
         return (ParseError, (self.msg, self.type, self.value, self.context))
 

lib2to3/pytree.py

@@ -678,8 +678,7 @@ def generate_matches(self, nodes):
                     if self.name:
                         r[self.name] = nodes[:count]
                     yield count, r
-            finally:
-                sys.stderr = save_stderr
+            sys.stderr = save_stderr
 
     def _iterative_matches(self, nodes):
         """Helper to iteratively yield the matches."""

pythoscope/_util.c

@@ -0,0 +1,43 @@
+/* Implementation of utility functions that couldn't be done in pure Python. */
+
+#include <Python.h>
+#include <compile.h>
+#include <frameobject.h>
+
+/* Python 2.3 headers don't include genobject (defined in Include/genobject.h
+   in later versions). We only need to grab the gi_frame, so this definition
+   will do. */
+typedef struct {
+    PyObject_HEAD
+    PyFrameObject *gi_frame;
+} genobject;
+
+static PyObject *
+_generator_has_ended(PyObject *self, PyObject *args)
+{
+    genobject *gen;
+    PyFrameObject *frame;
+
+    if (!PyArg_ParseTuple(args, "O", &gen))
+        return NULL;
+    /* Check if gen is a generator done on the Python level. */
+
+    frame = gen->gi_frame;
+
+    /* Python 2.5 releases gi_frame once the generator is done, so it has to be
+       checked first.
+       Earlier Pythons leave gi_frame intact, so the f_stacktop pointer
+       determines whether the generator is still running. */
+    return PyBool_FromLong(frame == NULL || frame->f_stacktop == NULL);
+}
+
+static PyMethodDef UtilMethods[] = {
+    {"_generator_has_ended",  _generator_has_ended, METH_VARARGS, NULL},
+    {NULL, NULL, 0, NULL}
+};
+
+PyMODINIT_FUNC
+init_util(void)
+{
+    (void) Py_InitModule("_util", UtilMethods);
+}

pythoscope/generator/__init__.py

@@ -8,7 +8,7 @@
 from pythoscope.store import Class, Function, FunctionCall, TestClass, \
     TestMethod, ModuleNotFound, UserObject, MethodCall, Method, Project, \
     GeneratorObject
-from pythoscope.util import camelize, compact, counted, flatten, \
+from pythoscope.util import any, camelize, compact, counted, flatten, \
     key_for_value, pluralize, set, sorted, underscore, union
 
 
@@ -120,7 +120,9 @@ def constructor_as_string(object, assigned_names={}):
     elif isinstance(object, CompositeObject):
         try:
             reconstructors, imports, uncomplete = zip(*get_contained_objects_info(object, assigned_names))
-        except ValueError:
+        # In Python <= 2.3 zip can raise TypeError if no arguments are provided.
+        # All Pythons can raise ValueError because of the wrong unpacking.
+        except (ValueError, TypeError):
             reconstructors, imports, uncomplete = [], [], []
         return CallString(object.constructor_format % ', '.join(reconstructors),
                           imports=union(object.imports, *imports),

pythoscope/inspector/__init__.py

@@ -1,7 +1,8 @@
 from pythoscope.inspector import static, dynamic
 from pythoscope.logger import log
 from pythoscope.store import ModuleNotFound
-from pythoscope.util import last_traceback, python_modules_below
+from pythoscope.util import generator_has_ended, last_traceback, \
+    python_modules_below
 
 
 def inspect_project(project):
@@ -55,6 +56,11 @@ def inspect_project_statically(project):
         add_and_update_points_of_entry(project)
 
 def inspect_project_dynamically(project):
+    if project.points_of_entry and hasattr(generator_has_ended, 'unreliable'):
+        log.warning("Pure Python implementation of util.generator_has_ended is "
+                    "not reliable on Python 2.4 and lower. Please compile the "
+                    "_util module or use Python 2.5 or higher.")
+
     for poe in project.points_of_entry.values():
         try:
             log.info("Inspecting point of entry %s." % poe.name)

pythoscope/logger.py

@@ -1,7 +1,7 @@
 """This module defines the logging system.
 
-To change the logging level, assign INFO, DEBUG or ERROR to log.level. Default
-is INFO.
+To change the logging level, assign DEBUG, ERROR, INFO or WARNING to log.level.
+Default is INFO.
 
 To change the output stream, call the set_output() function. Default is
 sys.stderr.
@@ -16,9 +16,10 @@
 from pythoscope.util import module_path_to_name
 
 
-INFO  = logging.INFO
 DEBUG = logging.DEBUG
 ERROR = logging.ERROR
+INFO  = logging.INFO
+WARNING = logging.WARNING
 
 def path2modname(path, default=""):
     """Take a path to a pythoscope module and return a module name in dot-style

pythoscope/serializer.py

@@ -4,12 +4,12 @@
 import sets
 import types
 
-from pythoscope.util import RePatternType, all, class_name, frozenset, \
-    module_name, regexp_flags_as_string, set, string2id, underscore
+from pythoscope.util import RePatternType, all, class_name, class_of, \
+    frozenset, module_name, regexp_flags_as_string, set, string2id, underscore
 
 
 # Filter out private attributes, like __doc__, __name__ and __package__.
-BUILTIN_EXCEPTION_TYPES = set(v for k,v in exceptions.__dict__.items() if not k.startswith('_'))
+BUILTIN_EXCEPTION_TYPES = set([v for k,v in exceptions.__dict__.items() if not k.startswith('_')])
 
 # Exceptions with special semantics for the `args` attribute.
 # See <http://docs.python.org/library/exceptions.html#exceptions.EnvironmentError>
@@ -42,19 +42,19 @@ def get_human_readable_id(obj):
         return 'false'
 
     # ... based on object's type,
-    objtype = type(obj)
+    objclass = class_of(obj)
     mapping = {list: 'list',
                dict: 'dict',
                tuple: 'tuple',
                unicode: 'unicode_string',
                types.GeneratorType: 'generator'}
-    objid = mapping.get(objtype)
+    objid = mapping.get(objclass)
     if objid:
         return objid
 
     # ... or based on its supertype.
     if isinstance(obj, Exception):
-        return underscore(objtype.__name__)
+        return underscore(objclass.__name__)
     elif isinstance(obj, RePatternType):
         return "%s_pattern" % string2id(obj.pattern)
     elif isinstance(obj, types.FunctionType):
@@ -69,7 +69,7 @@ def get_human_readable_id(obj):
             string = "<>"
         # Looks like an instance without a custom __str__ defined.
         if string.startswith("<"):
-            return "%s_instance" % underscore(objtype.__name__)
+            return "%s_instance" % underscore(objclass.__name__)
         else:
             return string2id(string)
 
@@ -160,15 +160,23 @@ def __hash__(self):
         return hash(self.reconstructor)
 
     def __repr__(self):
-        return "ImmutableObject(%r, imports=%r)" % (self.reconstructor, self.imports)
+        if self.imports:
+            return "ImmutableObject(%r, imports=%r)" % (self.reconstructor, self.imports)
+        return "ImmutableObject(%r)" % self.reconstructor
 
     # :: object -> (string, set)
     def get_reconstructor_with_imports(obj):
         """
-        >>> ImmutableObject.get_reconstructor_with_imports(re.compile('abcd'))
-        ("re.compile('abcd')", set(['re']))
-        >>> ImmutableObject.get_reconstructor_with_imports(re.compile('abcd', re.I | re.M))
-        ("re.compile('abcd', re.IGNORECASE | re.MULTILINE)", set(['re']))
+        >>> reconstructor, imports = ImmutableObject.get_reconstructor_with_imports(re.compile('abcd'))
+        >>> reconstructor
+        "re.compile('abcd')"
+        >>> imports == set(['re'])
+        True
+        >>> reconstructor, imports = ImmutableObject.get_reconstructor_with_imports(re.compile('abcd', re.I | re.M))
+        >>> reconstructor
+        "re.compile('abcd', re.IGNORECASE | re.MULTILINE)"
+        >>> imports == set(['re'])
+        True
         """
         if isinstance(obj, (int, long, float, str, unicode, types.NoneType)):
             # Bultin types has very convienient representation.
@@ -261,6 +269,9 @@ def __init__(self, obj, serialize):
             self.constructor_format = self.type_formats_with_imports[type(obj)][0]
             self.imports = self.type_formats_with_imports[type(obj)][1]
 
+    def __repr__(self):
+        return "SequenceObject(%s)" % (self.constructor_format % self.contained_objects)
+
 class MapObject(CompositeObject):
     """A mutable object that contains unordered mapping of key/value pairs.
     """
@@ -289,7 +300,7 @@ def __init__(self, obj, serialize):
         self.constructor_format = "%s(%%s)" % class_name(obj)
         self.imports = set()
 
-        if type(obj) in BUILTIN_ENVIRONMENT_ERROR_TYPES and obj.filename is not None:
+        if class_of(obj) in BUILTIN_ENVIRONMENT_ERROR_TYPES and obj.filename is not None:
             self.args.append(serialize(obj.filename))
 
 def is_immutable(obj):
@@ -313,7 +324,7 @@ def is_builtin_exception(obj):
     NameError or EOFError. Return False for instances of user-defined
     exceptions.
     """
-    return type(obj) in BUILTIN_EXCEPTION_TYPES
+    return class_of(obj) in BUILTIN_EXCEPTION_TYPES
 
 def is_serialized_string(obj):
     return isinstance(obj, ImmutableObject) and obj.type_name == 'str'

pythoscope/store.py

@@ -15,8 +15,9 @@
 from pythoscope.util import all_of_type, set, module_path_to_name, \
      write_content_to_file, ensure_directory, DirectoryException, \
      get_last_modification_time, read_file_contents, is_generator_code, \
-     extract_subpath, directories_under, findfirst, contains_active_generator, \
-     map_values, class_name, module_name, starts_with_path, string2filename
+     extract_subpath, directories_under, findfirst, generator_has_ended, \
+     map_values, class_name, module_name, starts_with_path, string2filename, \
+     get_generator_from_frame
 
 
 CREATIONAL_METHODS = ['__init__', '__new__']
@@ -1048,7 +1049,7 @@ def save(self):
             try:
                 self.write(self.get_content())
             except DirectoryException, err:
-                raise ModuleSaveError(self.subpath, err.message)
+                raise ModuleSaveError(self.subpath, err.args[0])
             self.changed = False
 
 def object_id(obj):
@@ -1165,11 +1166,11 @@ def create_generator_object(self, definition, sargs, frame):
         gobject = GeneratorObject(definition, sargs)
         # Generator objects return None to the tracer when stopped. That
         # extra None we have to filter out manually (see
-        # _fix_generator_objects method). The only way to distinguish
-        # between active and stopped generators is to ask garbage collector
-        # about them. So we temporarily save the generator frame inside the
-        # GeneratorObject, so it can be inspected later.
-        gobject._frame = frame
+        # _fix_generator_objects method). We distinguish between active
+        # and stopped generators using the generator_has_ended() function.
+        # It needs the generator object itself, so we save it for later
+        # inspection inside the GeneratorObject.
+        gobject._generator = get_generator_from_frame(frame)
         return gobject
 
     # :: (type, Definition, Callable, args, code, frame) -> Call
@@ -1239,13 +1240,12 @@ def _fix_generator_objects(self):
         just bogus Nones placed on generator stop.
         """
         for gobject in self.iter_captured_generator_objects():
-            if not contains_active_generator(gobject._frame) \
+            if generator_has_ended(gobject._generator) \
                    and gobject.output \
                    and gobject.output[-1] == ImmutableObject(None):
                 gobject.output.pop()
-            # Once we know if the generator is active or not, we can discard
-            # the frame.
-            del gobject._frame
+            # Once we know if the generator is active or not, we can discard it.
+            del gobject._generator
 
 class PointOfEntry(Localizable):
     """Piece of code provided by the user that allows dynamic analysis.

pythoscope/tracer.py

@@ -5,7 +5,9 @@
 from pythoscope.util import compact
 
 
-IGNORED_NAMES = ["<module>", "<genexpr>"]
+# Pythons <= 2.4 surround `exec`uted code with a block named "?",
+# while Pythons > 2.4 use "<module>".
+IGNORED_NAMES = ["?", "<module>", "<genexpr>"]
 
 def find_variable(frame, varname):
     """Find variable named varname in the scope of a frame.
@@ -106,11 +108,18 @@ def function():
         return function
     return code
 
-class Tracer(object):
+def is_generator_exit(obj):
+    try:
+        return obj is GeneratorExit
+    # Pythons 2.4 and lower don't have GeneratorExit exceptions at all.
+    except NameError:
+        return False
+
+class StandardTracer(object):
     """Wrapper around basic C{sys.settrace} mechanism that maps 'call', 'return'
     and 'exception' events into more meaningful callbacks.
 
-    See L{ICallback} for details on events that Tracer reports.
+    See L{ICallback} for details on events that tracer reports.
     """
     def __init__(self, callback):
         self.callback = callback
@@ -123,7 +132,7 @@ def trace(self, code):
         """Trace execution of given code. Code may be either a function
         or a string with Python code.
 
-        This method may be invoked many times for a single Tracer instance.
+        This method may be invoked many times for a single tracer instance.
         """
         self.setup(code)
         sys.settrace(self.tracer)
@@ -156,7 +165,7 @@ def tracer(self, frame, event, arg):
         elif event == 'return':
             self.callback.returned(arg)
         elif event == 'exception':
-            if arg[0] is not GeneratorExit:
+            if not is_generator_exit(arg[0]):
                 # There are three cases here, each requiring different handling
                 # of values in arg[0] and arg[1]. First, we may get a regular
                 # exception generated by the `raise` statement. Second, we may
@@ -220,6 +229,39 @@ def record_call(self, frame):
             input = input_from_argvalues(*inspect.getargvalues(frame))
             return self.callback.function_called(name, input, code, frame)
 
+class Python23Tracer(StandardTracer):
+    """Version of the tracer working around a subtle difference in exception
+    handling of Python 2.3.
+
+    In Python 2.4 and higher, when a function (or method) exits with
+    an exception, interpreter reports two events to a trace function:
+    first 'exception' and then 'return' right after that.
+
+    In Python 2.3 the second event isn't reported, i.e. only 'exception'
+    events are passed to a trace function. For the sake of consistency this
+    version of the tracer will inject a 'return' event before each consecutive
+    exception reported.
+    """
+    def __init__(self, *args):
+        super(Python23Tracer, self).__init__(*args)
+        self.propagating_exception = False
+
+    def tracer(self, frame, event, arg):
+        if event == 'exception':
+            if self.propagating_exception:
+                self.callback.returned(None)
+            else:
+                self.propagating_exception = True
+        elif event in ['call', 'return']:
+            self.propagating_exception = False
+        return super(Python23Tracer, self).tracer(frame, event, arg)
+
+def Tracer(*args):
+    if sys.version_info < (2, 4):
+        return Python23Tracer(*args)
+    else:
+        return StandardTracer(*args)
+
 class ICallback(object):
     """Interface that Tracer's callback object should adhere to.
     """