Move instruction decoding from VM -> Frame.

- Misc refactoring and comments.
- Also, Minimize the generator_exception test case.

There are still 4 Oil unit tests failures under byterun.

opy/byterun/pyobj.py

@@ -2,6 +2,7 @@
from __future__ import print_function

import collections
import dis
import inspect
import sys
import types
@@ -23,13 +24,13 @@ class Function(object):
        '_vm', '_func',
    ]

    def __init__(self, name, code, globs, defaults, closure, vm):
    def __init__(self, name, code, globs, locs, defaults, closure, vm):
        self._vm = vm
        self.func_code = code
        self.func_name = self.__name__ = name or code.co_name
        self.func_defaults = tuple(defaults)
        self.func_globals = globs
        self.func_locals = self._vm.frame.f_locals
        self.func_locals = locs
        self.__dict__ = {}
        self.func_closure = closure
        self.__doc__ = code.co_consts[0] if code.co_consts else None
@@ -73,9 +74,9 @@ def __call__(self, *args, **kwargs):
            callargs = inspect.getcallargs(self._func, *args, **kwargs)
        #print('-- func_name %s CALLS ARGS %s' % (self.func_name, callargs))

        frame = self._vm.make_frame(
            self.func_code, callargs, self.func_globals, {}
        )
        frame = self._vm.make_frame(self.func_code, callargs,
                                    self.func_globals, {})

        CO_GENERATOR = 32           # flag for "this code uses yield"
        if self.func_code.co_flags & CO_GENERATOR:
            gen = Generator(frame, self._vm)
@@ -285,6 +286,43 @@ def handle_block_stack(self, why, vm):

        return why

    def decode_next(self):
        """
        Parse 1 - 3 bytes of bytecode into an instruction and maybe arguments.
        """
        byteCode = ord(self.f_code.co_code[self.f_lasti])
        self.f_lasti += 1
        byteName = dis.opname[byteCode]
        arg = None
        arguments = []

        if byteCode >= dis.HAVE_ARGUMENT:
            arg = self.f_code.co_code[self.f_lasti : self.f_lasti+2]
            self.f_lasti += 2
            intArg = ord(arg[0]) + (ord(arg[1]) << 8)
            if byteCode in dis.hasconst:
                arg = self.f_code.co_consts[intArg]
            elif byteCode in dis.hasfree:
                if intArg < len(self.f_code.co_cellvars):
                    arg = self.f_code.co_cellvars[intArg]
                else:
                    var_idx = intArg - len(self.f_code.co_cellvars)
                    arg = self.f_code.co_freevars[var_idx]
            elif byteCode in dis.hasname:
                arg = self.f_code.co_names[intArg]
            elif byteCode in dis.hasjrel:
                arg = self.f_lasti + intArg
            elif byteCode in dis.hasjabs:
                arg = intArg
            elif byteCode in dis.haslocal:
                arg = self.f_code.co_varnames[intArg]
            else:
                arg = intArg
            arguments = [arg]

        return byteName, arguments


    def line_number(self):
        """Get the current line number the frame is executing."""
        # We don't keep f_lineno up to date, so calculate it based on the
@@ -311,7 +349,7 @@ def line_number(self):
class Generator(object):
    def __init__(self, g_frame, vm):
        self.gi_frame = g_frame
        self.vm = vm
        self._vm = vm
        self.started = False
        self.finished = False

@@ -326,7 +364,7 @@ def send(self, value=None):
            raise TypeError("Can't send non-None value to a just-started generator")
        self.gi_frame.stack.append(value)
        self.started = True
        val = self.vm.resume_frame(self.gi_frame)
        val = self._vm.resume_frame(self.gi_frame)
        if self.finished:
            raise StopIteration(val)
        return val

opy/byterun/pyvm2.py

@@ -117,6 +117,9 @@ def jump(self, offset):
        self.frame.jump(offset)

    def make_frame(self, code, callargs={}, f_globals=None, f_locals=None):
        """
        Called by self.run_code and Function.__call__.
        """
        log.info("make_frame: code=%r, callargs=%s" % (code, repper(callargs)))
        if f_globals is not None:
            f_globals = f_globals
@@ -136,24 +139,15 @@ def make_frame(self, code, callargs={}, f_globals=None, f_locals=None):
        frame = Frame(code, f_globals, f_locals, self.frame)
        return frame

    def push_frame(self, frame):
        self.frames.append(frame)
        self.frame = frame

    def pop_frame(self):
        self.frames.pop()
        if self.frames:
            self.frame = self.frames[-1]
        else:
            self.frame = None

    def resume_frame(self, frame):
        """Called by Generator."""
        frame.f_back = self.frame
        val = self.run_frame(frame)
        frame.f_back = None
        return val

    def run_code(self, code, f_globals=None, f_locals=None):
        """Main entry point."""
        frame = self.make_frame(code, f_globals=f_globals, f_locals=f_locals)
        val = self.run_frame(frame)
        # Check some invariants
@@ -164,43 +158,6 @@ def run_code(self, code, f_globals=None, f_locals=None):

        return val

    def parse_byte_and_args(self):
        """ Parse 1 - 3 bytes of bytecode into
        an instruction and optionally arguments."""
        f = self.frame
        opoffset = f.f_lasti
        byteCode = ord(f.f_code.co_code[opoffset])
        f.f_lasti += 1
        byteName = dis.opname[byteCode]
        arg = None
        arguments = []

        if byteCode >= dis.HAVE_ARGUMENT:
            arg = f.f_code.co_code[f.f_lasti:f.f_lasti+2]
            f.f_lasti += 2
            intArg = ord(arg[0]) + (ord(arg[1]) << 8)
            if byteCode in dis.hasconst:
                arg = f.f_code.co_consts[intArg]
            elif byteCode in dis.hasfree:
                if intArg < len(f.f_code.co_cellvars):
                    arg = f.f_code.co_cellvars[intArg]
                else:
                    var_idx = intArg - len(f.f_code.co_cellvars)
                    arg = f.f_code.co_freevars[var_idx]
            elif byteCode in dis.hasname:
                arg = f.f_code.co_names[intArg]
            elif byteCode in dis.hasjrel:
                arg = f.f_lasti + intArg
            elif byteCode in dis.hasjabs:
                arg = intArg
            elif byteCode in dis.haslocal:
                arg = f.f_code.co_varnames[intArg]
            else:
                arg = intArg
            arguments = [arg]

        return byteName, arguments, opoffset

    def logTick(self, byteName, arguments, opoffset, linestarts):
        """ Log arguments, block stack, and data stack for each opcode."""
        indent = "    " * (len(self.frames)-1)
@@ -254,6 +211,18 @@ def dispatch(self, byteName, arguments):

        return why

    # Helpers for run_frame
    def _push_frame(self, frame):
        self.frames.append(frame)
        self.frame = frame

    def _pop_frame(self):
        self.frames.pop()
        if self.frames:
            self.frame = self.frames[-1]
        else:
            self.frame = None

    def run_frame(self, frame):
        """Run a frame until it returns (somehow).

@@ -266,12 +235,13 @@ def run_frame(self, frame):
        linestarts = dict(dis.findlinestarts(frame.f_code))
        #print('STARTS %s ' % linestarts)

        self.push_frame(frame)
        self._push_frame(frame)
        num_ticks = 0
        while True:
            num_ticks += 1

            byteName, arguments, opoffset = self.parse_byte_and_args()
            opoffset = self.frame.f_lasti  # For logging only
            byteName, arguments = self.frame.decode_next()
            if self.verbose:
                self.logTick(byteName, arguments, opoffset, linestarts)

@@ -296,7 +266,7 @@ def run_frame(self, frame):

        # TODO: handle generator exception state

        self.pop_frame()
        self._pop_frame()

        if why == 'exception':
            exctype, value, tb = self.last_exception
@@ -796,7 +766,8 @@ def byte_MAKE_FUNCTION(self, argc):
        code = self.pop()
        defaults = self.popn(argc)
        globs = self.frame.f_globals
        fn = Function(name, code, globs, defaults, None, self)
        locs = self.frame.f_locals
        fn = Function(name, code, globs, locs, defaults, None, self)
        self.push(fn)

    def byte_LOAD_CLOSURE(self, name):
@@ -807,7 +778,8 @@ def byte_MAKE_CLOSURE(self, argc):
        closure, code = self.popn(2)
        defaults = self.popn(argc)
        globs = self.frame.f_globals
        fn = Function(name, code, globs, defaults, closure, self)
        locs = self.frame.f_locals
        fn = Function(name, code, globs, locs, defaults, closure, self)
        self.push(fn)

    def byte_CALL_FUNCTION(self, arg):
@@ -893,6 +865,7 @@ def call_function(self, arg, args, kwargs):
            defaults = func.func_defaults or ()
            byterun_func = Function(
                    func.func_name, func.func_code, func.func_globals,
                    self.frame.f_locals,  # Is this right?  How does it work?
                    defaults, func.func_closure, self)
        else:
            byterun_func = func

opy/testdata/generator_exception.py

@@ -12,6 +12,19 @@ def Tokenize(s):
    yield item


class TokenizeClass(object):  # NOT a generator
  def __init__(self, s):
    self.s = s
    self.i = 1

  def next(self):
    if self.i == 4:
      raise StopIteration()
    ret = str(self.i)
    self.i += 1
    return ret


class Parser(object):
  """Recursive TDOP parser."""

@@ -29,10 +42,13 @@ def Next(self):


def main(argv):
  lexer = Tokenize('1+2')
  if 1:
    lexer = Tokenize('1+2')  # does NOT work
  else:
    lexer = TokenizeClass('1+2')  # WORKS

  p = Parser(lexer)
  p.Next()
  p.Next()
  print('Done')