Skip to content
Browse files

Merged in support-python2.3 branch.

------------------------------------------------------------
Use --include-merged or -n0 to see merged revisions.
  • Loading branch information...
1 parent fdc3f63 commit 1a1f1468baf6f4bda1dc59bc93684ff67383f470 @mkwiatkowski committed with
View
2 .bzrignore
@@ -1,3 +1,5 @@
pythoscope.egg-info
lib2to3/Grammar*.pickle
lib2to3/PatternGrammar*.pickle
+build/
+dist/
View
7 lib2to3/pgen2/parse.py
@@ -34,8 +34,11 @@ def __reduce__(self):
"""Implemented so pickle can serialize this object.
>>> import pickle
- >>> pickle.loads(pickle.dumps(ParseError(1, 2, 3, 4)))
- ParseError('1: type=2, value=3, context=4',)
+ >>> pe = pickle.loads(pickle.dumps(ParseError(1, 2, 3, 4)))
+ >>> isinstance(pe, ParseError)
+ True
+ >>> (pe.msg, pe.type, pe.value, pe.context) == (1, 2, 3, 4)
+ True
"""
return (ParseError, (self.msg, self.type, self.value, self.context))
View
3 lib2to3/pytree.py
@@ -678,8 +678,7 @@ def generate_matches(self, nodes):
if self.name:
r[self.name] = nodes[:count]
yield count, r
- finally:
- sys.stderr = save_stderr
+ sys.stderr = save_stderr
def _iterative_matches(self, nodes):
"""Helper to iteratively yield the matches."""
View
43 pythoscope/_util.c
@@ -0,0 +1,43 @@
+/* Implementation of utility functions that couldn't be done in pure Python. */
+
+#include <Python.h>
+#include <compile.h>
+#include <frameobject.h>
+
+/* Python 2.3 headers don't include genobject (defined in Include/genobject.h
+ in later versions). We only need to grab the gi_frame, so this definition
+ will do. */
+typedef struct {
+ PyObject_HEAD
+ PyFrameObject *gi_frame;
+} genobject;
+
+static PyObject *
+_generator_has_ended(PyObject *self, PyObject *args)
+{
+ genobject *gen;
+ PyFrameObject *frame;
+
+ if (!PyArg_ParseTuple(args, "O", &gen))
+ return NULL;
+ /* Check if gen is a generator done on the Python level. */
+
+ frame = gen->gi_frame;
+
+ /* Python 2.5 releases gi_frame once the generator is done, so it has to be
+ checked first.
+ Earlier Pythons leave gi_frame intact, so the f_stacktop pointer
+ determines whether the generator is still running. */
+ return PyBool_FromLong(frame == NULL || frame->f_stacktop == NULL);
+}
+
+static PyMethodDef UtilMethods[] = {
+ {"_generator_has_ended", _generator_has_ended, METH_VARARGS, NULL},
+ {NULL, NULL, 0, NULL}
+};
+
+PyMODINIT_FUNC
+init_util(void)
+{
+ (void) Py_InitModule("_util", UtilMethods);
+}
View
6 pythoscope/generator/__init__.py
@@ -8,7 +8,7 @@
from pythoscope.store import Class, Function, FunctionCall, TestClass, \
TestMethod, ModuleNotFound, UserObject, MethodCall, Method, Project, \
GeneratorObject
-from pythoscope.util import camelize, compact, counted, flatten, \
+from pythoscope.util import any, camelize, compact, counted, flatten, \
key_for_value, pluralize, set, sorted, underscore, union
@@ -120,7 +120,9 @@ def constructor_as_string(object, assigned_names={}):
elif isinstance(object, CompositeObject):
try:
reconstructors, imports, uncomplete = zip(*get_contained_objects_info(object, assigned_names))
- except ValueError:
+ # In Python <= 2.3 zip can raise TypeError if no arguments are provided.
+ # All Pythons can raise ValueError because of the wrong unpacking.
+ except (ValueError, TypeError):
reconstructors, imports, uncomplete = [], [], []
return CallString(object.constructor_format % ', '.join(reconstructors),
imports=union(object.imports, *imports),
View
8 pythoscope/inspector/__init__.py
@@ -1,7 +1,8 @@
from pythoscope.inspector import static, dynamic
from pythoscope.logger import log
from pythoscope.store import ModuleNotFound
-from pythoscope.util import last_traceback, python_modules_below
+from pythoscope.util import generator_has_ended, last_traceback, \
+ python_modules_below
def inspect_project(project):
@@ -55,6 +56,11 @@ def inspect_project_statically(project):
add_and_update_points_of_entry(project)
def inspect_project_dynamically(project):
+ if project.points_of_entry and hasattr(generator_has_ended, 'unreliable'):
+ log.warning("Pure Python implementation of util.generator_has_ended is "
+ "not reliable on Python 2.4 and lower. Please compile the "
+ "_util module or use Python 2.5 or higher.")
+
for poe in project.points_of_entry.values():
try:
log.info("Inspecting point of entry %s." % poe.name)
View
7 pythoscope/logger.py
@@ -1,7 +1,7 @@
"""This module defines the logging system.
-To change the logging level, assign INFO, DEBUG or ERROR to log.level. Default
-is INFO.
+To change the logging level, assign DEBUG, ERROR, INFO or WARNING to log.level.
+Default is INFO.
To change the output stream, call the set_output() function. Default is
sys.stderr.
@@ -16,9 +16,10 @@
from pythoscope.util import module_path_to_name
-INFO = logging.INFO
DEBUG = logging.DEBUG
ERROR = logging.ERROR
+INFO = logging.INFO
+WARNING = logging.WARNING
def path2modname(path, default=""):
"""Take a path to a pythoscope module and return a module name in dot-style
View
39 pythoscope/serializer.py
@@ -4,12 +4,12 @@
import sets
import types
-from pythoscope.util import RePatternType, all, class_name, frozenset, \
- module_name, regexp_flags_as_string, set, string2id, underscore
+from pythoscope.util import RePatternType, all, class_name, class_of, \
+ frozenset, module_name, regexp_flags_as_string, set, string2id, underscore
# Filter out private attributes, like __doc__, __name__ and __package__.
-BUILTIN_EXCEPTION_TYPES = set(v for k,v in exceptions.__dict__.items() if not k.startswith('_'))
+BUILTIN_EXCEPTION_TYPES = set([v for k,v in exceptions.__dict__.items() if not k.startswith('_')])
# Exceptions with special semantics for the `args` attribute.
# See <http://docs.python.org/library/exceptions.html#exceptions.EnvironmentError>
@@ -42,19 +42,19 @@ def get_human_readable_id(obj):
return 'false'
# ... based on object's type,
- objtype = type(obj)
+ objclass = class_of(obj)
mapping = {list: 'list',
dict: 'dict',
tuple: 'tuple',
unicode: 'unicode_string',
types.GeneratorType: 'generator'}
- objid = mapping.get(objtype)
+ objid = mapping.get(objclass)
if objid:
return objid
# ... or based on its supertype.
if isinstance(obj, Exception):
- return underscore(objtype.__name__)
+ return underscore(objclass.__name__)
elif isinstance(obj, RePatternType):
return "%s_pattern" % string2id(obj.pattern)
elif isinstance(obj, types.FunctionType):
@@ -69,7 +69,7 @@ def get_human_readable_id(obj):
string = "<>"
# Looks like an instance without a custom __str__ defined.
if string.startswith("<"):
- return "%s_instance" % underscore(objtype.__name__)
+ return "%s_instance" % underscore(objclass.__name__)
else:
return string2id(string)
@@ -160,15 +160,23 @@ def __hash__(self):
return hash(self.reconstructor)
def __repr__(self):
- return "ImmutableObject(%r, imports=%r)" % (self.reconstructor, self.imports)
+ if self.imports:
+ return "ImmutableObject(%r, imports=%r)" % (self.reconstructor, self.imports)
+ return "ImmutableObject(%r)" % self.reconstructor
# :: object -> (string, set)
def get_reconstructor_with_imports(obj):
"""
- >>> ImmutableObject.get_reconstructor_with_imports(re.compile('abcd'))
- ("re.compile('abcd')", set(['re']))
- >>> ImmutableObject.get_reconstructor_with_imports(re.compile('abcd', re.I | re.M))
- ("re.compile('abcd', re.IGNORECASE | re.MULTILINE)", set(['re']))
+ >>> reconstructor, imports = ImmutableObject.get_reconstructor_with_imports(re.compile('abcd'))
+ >>> reconstructor
+ "re.compile('abcd')"
+ >>> imports == set(['re'])
+ True
+ >>> reconstructor, imports = ImmutableObject.get_reconstructor_with_imports(re.compile('abcd', re.I | re.M))
+ >>> reconstructor
+ "re.compile('abcd', re.IGNORECASE | re.MULTILINE)"
+ >>> imports == set(['re'])
+ True
"""
if isinstance(obj, (int, long, float, str, unicode, types.NoneType)):
# Bultin types has very convienient representation.
@@ -261,6 +269,9 @@ def __init__(self, obj, serialize):
self.constructor_format = self.type_formats_with_imports[type(obj)][0]
self.imports = self.type_formats_with_imports[type(obj)][1]
+ def __repr__(self):
+ return "SequenceObject(%s)" % (self.constructor_format % self.contained_objects)
+
class MapObject(CompositeObject):
"""A mutable object that contains unordered mapping of key/value pairs.
"""
@@ -289,7 +300,7 @@ def __init__(self, obj, serialize):
self.constructor_format = "%s(%%s)" % class_name(obj)
self.imports = set()
- if type(obj) in BUILTIN_ENVIRONMENT_ERROR_TYPES and obj.filename is not None:
+ if class_of(obj) in BUILTIN_ENVIRONMENT_ERROR_TYPES and obj.filename is not None:
self.args.append(serialize(obj.filename))
def is_immutable(obj):
@@ -313,7 +324,7 @@ def is_builtin_exception(obj):
NameError or EOFError. Return False for instances of user-defined
exceptions.
"""
- return type(obj) in BUILTIN_EXCEPTION_TYPES
+ return class_of(obj) in BUILTIN_EXCEPTION_TYPES
def is_serialized_string(obj):
return isinstance(obj, ImmutableObject) and obj.type_name == 'str'
View
24 pythoscope/store.py
@@ -15,8 +15,9 @@
from pythoscope.util import all_of_type, set, module_path_to_name, \
write_content_to_file, ensure_directory, DirectoryException, \
get_last_modification_time, read_file_contents, is_generator_code, \
- extract_subpath, directories_under, findfirst, contains_active_generator, \
- map_values, class_name, module_name, starts_with_path, string2filename
+ extract_subpath, directories_under, findfirst, generator_has_ended, \
+ map_values, class_name, module_name, starts_with_path, string2filename, \
+ get_generator_from_frame
CREATIONAL_METHODS = ['__init__', '__new__']
@@ -1048,7 +1049,7 @@ def save(self):
try:
self.write(self.get_content())
except DirectoryException, err:
- raise ModuleSaveError(self.subpath, err.message)
+ raise ModuleSaveError(self.subpath, err.args[0])
self.changed = False
def object_id(obj):
@@ -1165,11 +1166,11 @@ def create_generator_object(self, definition, sargs, frame):
gobject = GeneratorObject(definition, sargs)
# Generator objects return None to the tracer when stopped. That
# extra None we have to filter out manually (see
- # _fix_generator_objects method). The only way to distinguish
- # between active and stopped generators is to ask garbage collector
- # about them. So we temporarily save the generator frame inside the
- # GeneratorObject, so it can be inspected later.
- gobject._frame = frame
+ # _fix_generator_objects method). We distinguish between active
+ # and stopped generators using the generator_has_ended() function.
+ # It needs the generator object itself, so we save it for later
+ # inspection inside the GeneratorObject.
+ gobject._generator = get_generator_from_frame(frame)
return gobject
# :: (type, Definition, Callable, args, code, frame) -> Call
@@ -1239,13 +1240,12 @@ def _fix_generator_objects(self):
just bogus Nones placed on generator stop.
"""
for gobject in self.iter_captured_generator_objects():
- if not contains_active_generator(gobject._frame) \
+ if generator_has_ended(gobject._generator) \
and gobject.output \
and gobject.output[-1] == ImmutableObject(None):
gobject.output.pop()
- # Once we know if the generator is active or not, we can discard
- # the frame.
- del gobject._frame
+ # Once we know if the generator is active or not, we can discard it.
+ del gobject._generator
class PointOfEntry(Localizable):
"""Piece of code provided by the user that allows dynamic analysis.
View
52 pythoscope/tracer.py
@@ -5,7 +5,9 @@
from pythoscope.util import compact
-IGNORED_NAMES = ["<module>", "<genexpr>"]
+# Pythons <= 2.4 surround `exec`uted code with a block named "?",
+# while Pythons > 2.4 use "<module>".
+IGNORED_NAMES = ["?", "<module>", "<genexpr>"]
def find_variable(frame, varname):
"""Find variable named varname in the scope of a frame.
@@ -106,11 +108,18 @@ def function():
return function
return code
-class Tracer(object):
+def is_generator_exit(obj):
+ try:
+ return obj is GeneratorExit
+ # Pythons 2.4 and lower don't have GeneratorExit exceptions at all.
+ except NameError:
+ return False
+
+class StandardTracer(object):
"""Wrapper around basic C{sys.settrace} mechanism that maps 'call', 'return'
and 'exception' events into more meaningful callbacks.
- See L{ICallback} for details on events that Tracer reports.
+ See L{ICallback} for details on events that tracer reports.
"""
def __init__(self, callback):
self.callback = callback
@@ -123,7 +132,7 @@ def trace(self, code):
"""Trace execution of given code. Code may be either a function
or a string with Python code.
- This method may be invoked many times for a single Tracer instance.
+ This method may be invoked many times for a single tracer instance.
"""
self.setup(code)
sys.settrace(self.tracer)
@@ -156,7 +165,7 @@ def tracer(self, frame, event, arg):
elif event == 'return':
self.callback.returned(arg)
elif event == 'exception':
- if arg[0] is not GeneratorExit:
+ if not is_generator_exit(arg[0]):
# There are three cases here, each requiring different handling
# of values in arg[0] and arg[1]. First, we may get a regular
# exception generated by the `raise` statement. Second, we may
@@ -220,6 +229,39 @@ def record_call(self, frame):
input = input_from_argvalues(*inspect.getargvalues(frame))
return self.callback.function_called(name, input, code, frame)
+class Python23Tracer(StandardTracer):
+ """Version of the tracer working around a subtle difference in exception
+ handling of Python 2.3.
+
+ In Python 2.4 and higher, when a function (or method) exits with
+ an exception, interpreter reports two events to a trace function:
+ first 'exception' and then 'return' right after that.
+
+ In Python 2.3 the second event isn't reported, i.e. only 'exception'
+ events are passed to a trace function. For the sake of consistency this
+ version of the tracer will inject a 'return' event before each consecutive
+ exception reported.
+ """
+ def __init__(self, *args):
+ super(Python23Tracer, self).__init__(*args)
+ self.propagating_exception = False
+
+ def tracer(self, frame, event, arg):
+ if event == 'exception':
+ if self.propagating_exception:
+ self.callback.returned(None)
+ else:
+ self.propagating_exception = True
+ elif event in ['call', 'return']:
+ self.propagating_exception = False
+ return super(Python23Tracer, self).tracer(frame, event, arg)
+
+def Tracer(*args):
+ if sys.version_info < (2, 4):
+ return Python23Tracer(*args)
+ else:
+ return StandardTracer(*args)
+
class ICallback(object):
"""Interface that Tracer's callback object should adhere to.
"""
View
49 pythoscope/util.py
@@ -22,11 +22,11 @@
try:
sorted = sorted
except NameError:
- def sorted(iterable, cmp=cmp, key=None):
+ def sorted(iterable, compare=cmp, key=None):
if key:
- cmp = lambda x,y: cmp(key(x), key(y))
+ compare = lambda x,y: cmp(key(x), key(y))
alist = list(iterable)
- alist.sort(cmp)
+ alist.sort(compare)
return alist
try:
@@ -39,6 +39,15 @@ def all(iterable):
return True
try:
+ any = any
+except NameError:
+ def any(iterable):
+ for element in iterable:
+ if element:
+ return True
+ return False
+
+try:
from itertools import groupby
except ImportError:
# Code taken from http://docs.python.org/lib/itertools-functions.html .
@@ -318,12 +327,42 @@ def key_for_value(dictionary, value):
if v == value:
return k
-def contains_active_generator(frame):
- return bool(all_of_type(gc.get_referrers(frame), types.GeneratorType))
+def get_generator_from_frame(frame):
+ generators = all_of_type(gc.get_referrers(frame), types.GeneratorType)
+ if generators:
+ return generators[0]
def is_generator_code(code):
return code.co_flags & 0x20 != 0
+def generator_has_ended(generator):
+ """Return True if the generator has been exhausted and False otherwise.
+
+ >>> generator_has_ended(1)
+ Traceback (most recent call last):
+ ...
+ TypeError: argument is not a generator
+ """
+ if not isinstance(generator, types.GeneratorType):
+ raise TypeError("argument is not a generator")
+ return _generator_has_ended(generator)
+
+try:
+ from _util import _generator_has_ended
+except ImportError:
+ if sys.version_info < (2, 5):
+ # In Python 2.4 and earlier we can't reliably tell if a generator
+ # is active or not without going to the C level. We assume it
+ # has ended, as it will be true most of the time in our use case.
+ def _generator_has_ended(generator):
+ return True
+ generator_has_ended.unreliable = True
+ else:
+ # This is a hack that uses the fact that in Python 2.5 and higher
+ # generator frame is garbage collected once the generator has ended.
+ def _generator_has_ended(generator):
+ return generator.gi_frame is None
+
def compile_without_warnings(stmt):
"""Compile single interactive statement with Python interpreter warnings
disabled.
View
54 run-tests.py
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+import glob
+import shutil
+import sys
+
+from os import system as run
+from os import remove as rm
+
+def cp(src, dst):
+ shutil.copy(glob.glob(src)[0], dst)
+
+def main():
+ VERSIONS = [('2.3', ['tests', 'build']),
+ ('2.4', ['tests', 'build']),
+ ('2.5', ['tests'])]
+ results = {}
+
+ for ver, types in VERSIONS:
+ if 'tests' in types:
+ version = "%s-tests" % ver
+ print "*** Running tests on Python %s without binary modules." % ver
+ if run("nosetests-%s" % ver) == 0:
+ results[version] = 'OK'
+ else:
+ results[version] = 'FAIL (tests)'
+
+ if 'build' in types:
+ version = "%s-build" % ver
+ res1 = res2 = None
+ print "*** Running tests on Python %s with binary modules." % ver
+ res1 = run("python%s setup.py build -f" % ver)
+ if res1 == 0:
+ cp("build/lib.*-%s/pythoscope/_util.so" % ver, "pythoscope/")
+ res2 = run("nosetests-%s" % ver)
+ rm("pythoscope/_util.so")
+ if res1 == 0 and res2 == 0:
+ results[version] = 'OK'
+ else:
+ if res1 != 0:
+ results[version] = 'FAIL (compilation)'
+ else:
+ results[version] = 'FAIL (tests)'
+
+ print
+ for ver, result in sorted(results.iteritems()):
+ print "%s: %s" % (ver, result)
+
+ if [v for v in results.values() if v != 'OK']:
+ return 1
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(main())
View
12 setup.py
@@ -1,7 +1,17 @@
+import sys
+
+# Order is relevant, as setuptools monkeypatches distutils.
from setuptools import setup
+from distutils.core import Extension
from pythoscope import __version__
+# The C module doesn't need to be built for Python 2.5 and higher.
+if sys.version_info < (2, 5):
+ ext_modules = [Extension('pythoscope._util', sources=['pythoscope/_util.c'])]
+else:
+ ext_modules = []
+
setup(
name='pythoscope',
version=__version__,
@@ -13,6 +23,8 @@
license = 'MIT',
url = 'http://pythoscope.org',
+ ext_modules = ext_modules,
+
packages = ['pythoscope', 'pythoscope.inspector', 'pythoscope.generator', 'lib2to3', 'lib2to3.pgen2'],
package_data = {'pythoscope': [],
'lib2to3': ['*.txt']},
View
16 test/test_dynamic_inspector.py
@@ -8,7 +8,7 @@
SequenceObject, BuiltinException
from pythoscope.store import Class, Execution, Function, \
GeneratorObject, Method, UserObject, Project
-from pythoscope.util import findfirst
+from pythoscope.util import all, findfirst, generator_has_ended
from assertions import *
from helper import TestableProject, PointOfEntryMock, EmptyProjectExecution, \
@@ -656,6 +656,9 @@ def test_handles_single_yielded_value(self):
assert not gobject.raised_exception()
def test_handles_yielded_nones(self):
+ if hasattr(generator_has_ended, 'unreliable'):
+ raise SkipTest
+
gobject = inspect_returning_single_call(function_calling_generator_that_yields_none)
assert_generator_object({}, [None], gobject)
@@ -671,6 +674,9 @@ def test_handles_generator_objects_that_werent_destroyed(self):
assert_generator_object({}, [1], gobject)
def test_handles_generator_objects_that_yield_none_and_dont_get_destroyed(self):
+ if hasattr(generator_has_ended, 'unreliable'):
+ raise SkipTest
+
gobject = inspect_returning_single_call(function_calling_generator_that_yields_none_and_doesnt_get_destroyed)
assert_generator_object({}, [None], gobject)
@@ -789,7 +795,13 @@ def causes_interpreter_to_raise_syntax_error():
def raising_syntax_error(): exec 'a b c'
try: raising_syntax_error()
except: pass
- syntax_error_exc_args = ('invalid syntax', ('<string>', 1, 3, 'a b c'))
+ # Versions of Python up to 2.4 used None for a filename in syntax
+ # errors invoked by exec.
+ if sys.version_info >= (2, 5):
+ filename = '<string>'
+ else:
+ filename = None
+ syntax_error_exc_args = ('invalid syntax', (filename, 1, 3, 'a b c'))
call = inspect_returning_single_call(causes_interpreter_to_raise_syntax_error)
assert_call_with_exception({}, 'SyntaxError', call)
View
18 test/test_inspector.py
@@ -1,4 +1,7 @@
+from nose import SkipTest
+
from pythoscope.inspector import inspect_project
+from pythoscope.util import generator_has_ended
from assertions import *
from helper import CapturedLogger, P, ProjectInDirectory, TempDirectory
@@ -44,3 +47,18 @@ def test_reports_each_inspected_point_of_entry(self):
for path in paths:
assert_contains_once(self._get_log_output(),
"INFO: Inspecting point of entry %s." % path)
+
+ def test_warns_about_unreliable_implementation_of_util_generator_has_ended(self):
+ if not hasattr(generator_has_ended, 'unreliable'):
+ raise SkipTest
+
+ paths = ["edgar.py", "allan.py"]
+ project = ProjectInDirectory(self.tmpdir).with_points_of_entry(paths)
+
+ inspect_project(project)
+
+ assert_contains_once(self._get_log_output(),
+ "WARNING: Pure Python implementation of "
+ "util.generator_has_ended is not reliable on "
+ "Python 2.4 and lower. Please compile the _util "
+ "module or use Python 2.5 or higher.")

0 comments on commit 1a1f146

Please sign in to comment.
Something went wrong with that request. Please try again.