diff --git a/AUTHORS b/AUTHORS index 4680226d..9a042123 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Paul Swartz Ned Batchelder Allison Kaptur Laura Lindzey +Darius Bacon diff --git a/byterun/pyobj.py b/byterun/pyobj.py index f2924305..ef6995ef 100644 --- a/byterun/pyobj.py +++ b/byterun/pyobj.py @@ -2,6 +2,7 @@ import collections import inspect +import re import types import six @@ -23,7 +24,7 @@ def make_cell(value): class Function(object): __slots__ = [ 'func_code', 'func_name', 'func_defaults', 'func_globals', - 'func_locals', 'func_dict', 'func_closure', + 'func_dict', 'func_closure', '__name__', '__dict__', '__doc__', '_vm', '_func', ] @@ -34,7 +35,6 @@ def __init__(self, name, code, globs, defaults, closure, vm): self.func_name = self.__name__ = name or code.co_name self.func_defaults = tuple(defaults) self.func_globals = globs - self.func_locals = self._vm.frame.f_locals self.__dict__ = {} self.func_closure = closure self.__doc__ = code.co_consts[0] if code.co_consts else None @@ -61,17 +61,18 @@ def __get__(self, instance, owner): return self def __call__(self, *args, **kwargs): - if PY2 and self.func_name in ["", "", ""]: + if re.search(r'<(?:listcomp|setcomp|dictcomp|genexpr)>$', self.func_name): # D'oh! http://bugs.python.org/issue19611 Py2 doesn't know how to # inspect set comprehensions, dict comprehensions, or generator # expressions properly. They are always functions of one argument, - # so just do the right thing. + # so just do the right thing. Py3.4 also would fail without this + # hack, for list comprehensions too. (Haven't checked for other 3.x.) assert len(args) == 1 and not kwargs, "Surprising comprehension!" callargs = {".0": args[0]} else: callargs = inspect.getcallargs(self._func, *args, **kwargs) frame = self._vm.make_frame( - self.func_code, callargs, self.func_globals, {} + self.func_code, callargs, self.func_globals, {}, self.func_closure ) CO_GENERATOR = 32 # flag for "this code uses yield" if self.func_code.co_flags & CO_GENERATOR: @@ -135,7 +136,7 @@ def set(self, value): class Frame(object): - def __init__(self, f_code, f_globals, f_locals, f_back): + def __init__(self, f_code, f_globals, f_locals, f_closure, f_back): self.f_code = f_code self.f_globals = f_globals self.f_locals = f_locals @@ -151,24 +152,13 @@ def __init__(self, f_code, f_globals, f_locals, f_back): self.f_lineno = f_code.co_firstlineno self.f_lasti = 0 - if f_code.co_cellvars: - self.cells = {} - if not f_back.cells: - f_back.cells = {} - for var in f_code.co_cellvars: - # Make a cell for the variable in our locals, or None. - cell = Cell(self.f_locals.get(var)) - f_back.cells[var] = self.cells[var] = cell - else: - self.cells = None - + self.cells = {} if f_code.co_cellvars or f_code.co_freevars else None + for var in f_code.co_cellvars: + # Make a cell for the variable in our locals, or None. + self.cells[var] = Cell(self.f_locals.get(var)) if f_code.co_freevars: - if not self.cells: - self.cells = {} - for var in f_code.co_freevars: - assert self.cells is not None - assert f_back.cells, "f_back.cells: %r" % (f_back.cells,) - self.cells[var] = f_back.cells[var] + assert len(f_code.co_freevars) == len(f_closure) + self.cells.update(zip(f_code.co_freevars, f_closure)) self.block_stack = [] self.generator = None diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index 3e1a7277..b2fec891 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -15,7 +15,7 @@ PY3, PY2 = six.PY3, not six.PY3 -from .pyobj import Frame, Block, Method, Function, Generator +from .pyobj import Frame, Block, Method, Function, Generator, Cell log = logging.getLogger(__name__) @@ -90,7 +90,7 @@ def push_block(self, type, handler=None, level=None): def pop_block(self): return self.frame.block_stack.pop() - def make_frame(self, code, callargs={}, f_globals=None, f_locals=None): + def make_frame(self, code, callargs={}, f_globals=None, f_locals=None, f_closure=None): log.info("make_frame: code=%r, callargs=%s" % (code, repper(callargs))) if f_globals is not None: f_globals = f_globals @@ -107,7 +107,7 @@ def make_frame(self, code, callargs={}, f_globals=None, f_locals=None): '__package__': None, } f_locals.update(callargs) - frame = Frame(code, f_globals, f_locals, self.frame) + frame = Frame(code, f_globals, f_locals, f_closure, self.frame) return frame def push_frame(self, frame): @@ -421,7 +421,10 @@ def byte_LOAD_GLOBAL(self, name): elif name in f.f_builtins: val = f.f_builtins[name] else: - raise NameError("global name '%s' is not defined" % name) + if PY2: + raise NameError("global name '%s' is not defined" % name) + elif PY3: + raise NameError("name '%s' is not defined" % name) self.push(val) def byte_LOAD_DEREF(self, name): @@ -1029,7 +1032,7 @@ def byte_BUILD_CLASS(self): elif PY3: def byte_LOAD_BUILD_CLASS(self): # New in py3 - self.push(__build_class__) + self.push(build_class) def byte_STORE_LOCALS(self): self.frame.f_locals = self.pop() @@ -1037,3 +1040,51 @@ def byte_STORE_LOCALS(self): if 0: # Not in py2.7 def byte_SET_LINENO(self, lineno): self.frame.f_lineno = lineno + +if PY3: + def build_class(func, name, *bases, **kwds): + "Like __build_class__ in bltinmodule.c, but running in the byterun VM." + if not isinstance(func, Function): + raise TypeError("func must be a function") + if not isinstance(name, str): + raise TypeError("name is not a string") + metaclass = kwds.pop('metaclass', None) + # (We don't just write 'metaclass=None' in the signature above + # because that's a syntax error in Py2.) + if metaclass is None: + metaclass = type(bases[0]) if bases else type + if isinstance(metaclass, type): + metaclass = calculate_metaclass(metaclass, bases) + + try: + prepare = metaclass.__prepare__ + except AttributeError: + namespace = {} + else: + namespace = prepare(name, bases, **kwds) + + # Execute the body of func. This is the step that would go wrong if + # we tried to use the built-in __build_class__, because __build_class__ + # does not call func, it magically executes its body directly, as we + # do here (except we invoke our VirtualMachine instead of CPython's). + frame = func._vm.make_frame(func.func_code, + f_globals=func.func_globals, + f_locals=namespace, + f_closure=func.func_closure) + cell = func._vm.run_frame(frame) + + cls = metaclass(name, bases, namespace) + if isinstance(cell, Cell): + cell.set(cls) + return cls + + def calculate_metaclass(metaclass, bases): + "Determine the most derived metatype." + winner = metaclass + for base in bases: + t = type(base) + if issubclass(t, winner): + winner = t + elif not issubclass(winner, t): + raise TypeError("metaclass conflict", winner, t) + return winner diff --git a/tests/test_functions.py b/tests/test_functions.py index f86fd131..7df69df7 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -207,6 +207,18 @@ def f4(g): assert answer == 54 """) + def test_closure_vars_from_static_parent(self): + self.assert_ok("""\ + def f(xs): + return lambda: xs[0] + + def g(h): + xs = 5 + lambda: xs + return h() + + assert g(f([42])) == 42 + """) class TestGenerators(vmtest.VmTestCase): def test_first(self): diff --git a/tests/vmtest.py b/tests/vmtest.py index 9e071838..763aa174 100644 --- a/tests/vmtest.py +++ b/tests/vmtest.py @@ -40,6 +40,19 @@ def assert_ok(self, code, raises=None): # Print the disassembly so we'll see it if the test fails. dis_code(code) + # Run the code through our VM and the real Python interpreter, for comparison. + vm_value, vm_exc, vm_stdout = self.run_in_byterun(code) + py_value, py_exc, py_stdout = self.run_in_real_python(code) + + self.assert_same_exception(vm_exc, py_exc) + self.assertEqual(vm_stdout.getvalue(), py_stdout.getvalue()) + self.assertEqual(vm_value, py_value) + if raises: + self.assertIsInstance(vm_exc, raises) + else: + self.assertIsNone(vm_exc) + + def run_in_byterun(self, code): real_stdout = sys.stdout # Run the code through our VM. @@ -64,32 +77,36 @@ def assert_ok(self, code, raises=None): raise vm_exc = e finally: + sys.stdout = real_stdout real_stdout.write("-- stdout ----------\n") real_stdout.write(vm_stdout.getvalue()) - # Run the code through the real Python interpreter, for comparison. + return vm_value, vm_exc, vm_stdout + + def run_in_real_python(self, code): + real_stdout = sys.stdout py_stdout = six.StringIO() sys.stdout = py_stdout py_value = py_exc = None - globs = {} + globs = { + '__builtins__': __builtins__, + '__name__': '__main__', + '__doc__': None, + '__package__': None, + } + try: py_value = eval(code, globs, globs) except AssertionError: # pragma: no cover raise except Exception as e: py_exc = e + finally: + sys.stdout = real_stdout - sys.stdout = real_stdout - - self.assert_same_exception(vm_exc, py_exc) - self.assertEqual(vm_stdout.getvalue(), py_stdout.getvalue()) - self.assertEqual(vm_value, py_value) - if raises: - self.assertIsInstance(vm_exc, raises) - else: - self.assertIsNone(vm_exc) + return py_value, py_exc, py_stdout def assert_same_exception(self, e1, e2): """Exceptions don't implement __eq__, check it ourselves.""" diff --git a/tox.ini b/tox.ini index 4544cbc0..c7a5c858 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py33 +envlist = py27, py33, py34 [testenv] commands =